diff --git a/locales/en.json b/locales/en.json index 145156d1..a30d6f4f 100644 --- a/locales/en.json +++ b/locales/en.json @@ -810,6 +810,16 @@ "filmGrain": "Film Grain", "reset": "Reset to defaults" }, + "frameInterpolation": { + "title": "Frame Interpolation", + "hint": "Experimental neural frame interpolation (Framegen WebGPU runtime) that synthesizes extra frames between decoded stream frames for smoother motion. Adds display latency and GPU load. Web client mode only; uses shader-f16 when available.", + "nativeUnavailable": "Frame interpolation does not apply while native streaming renders the video outside the app window.", + "factor": "Factor", + "quality": "Model quality", + "qualityHint": "Internal resolution of interpolated frames. Lower uses less GPU; 480p is a good starting point.", + "weightsNotice": "Runtime code is MIT. Model weights are non-commercial research/personal use only (Framegen weight license).", + "reset": "Reset to defaults" + }, "codecDiagnostics": { "advanced": "Advanced - Codec Diagnostics", "title": "Codec Diagnostics", diff --git a/opennow-stable/.gitignore b/opennow-stable/.gitignore index 5152f10c..613e8ab4 100644 --- a/opennow-stable/.gitignore +++ b/opennow-stable/.gitignore @@ -3,3 +3,6 @@ dist dist-electron dist-release *.log + +# Framegen weights copied at postinstall (non-commercial; see third_party/framegen) +src/renderer/public/framegen-weights/ diff --git a/opennow-stable/electron.vite.config.ts b/opennow-stable/electron.vite.config.ts index 9c8f13f7..19910376 100644 --- a/opennow-stable/electron.vite.config.ts +++ b/opennow-stable/electron.vite.config.ts @@ -53,5 +53,8 @@ export default defineConfig({ "@shared": resolve("src/shared"), }, }, + optimizeDeps: { + include: ["framegen"], + }, }, }); diff --git a/opennow-stable/package-lock.json b/opennow-stable/package-lock.json index 0e0bd535..5fc0ba96 100644 --- a/opennow-stable/package-lock.json +++ b/opennow-stable/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "discord-rpc": "^4.0.1", "electron-updater": "^6.8.9", + "framegen": "^1.4.0", "posthog-node": "^5.48.1", "ws": "^8.21.3" }, @@ -25,6 +26,7 @@ "@types/three": "^0.185.4", "@types/ws": "^8.18.1", "@vitejs/plugin-react": "^5.2.0", + "@webgpu/types": "^0.1.71", "cross-env": "^10.1.0", "electron": "^43.3.0", "electron-builder": "^26.15.3", @@ -3793,6 +3795,13 @@ "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/@webgpu/types": { + "version": "0.1.71", + "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.71.tgz", + "integrity": "sha512-mMy8/ODcKhab808co15eW+yN+HgXoQxRQHTiBV9Mrvl1r0ufnid7YOcI+gi4eUWSWl9ezD6TW2KXccrL8HCh2A==", + "dev": true, + "license": "BSD-3-Clause" + }, "node_modules/@xmldom/xmldom": { "version": "0.8.13", "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.13.tgz", @@ -6059,6 +6068,15 @@ "node": ">= 6" } }, + "node_modules/framegen": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/framegen/-/framegen-1.4.0.tgz", + "integrity": "sha512-NqK6nckQMqW6l29mKlNzVlbwH89YLPi7wqiPbrmLSm3p9LMqWEnURTjubNQ8k8F4ogz2OWpygT5MSTmxtvojQQ==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/framer-motion": { "version": "12.43.0", "resolved": "https://registry.npmjs.org/framer-motion/-/framer-motion-12.43.0.tgz", diff --git a/opennow-stable/package.json b/opennow-stable/package.json index 3f17a276..954c4d57 100644 --- a/opennow-stable/package.json +++ b/opennow-stable/package.json @@ -19,7 +19,7 @@ "node": ">=22.22.0" }, "scripts": { - "postinstall": "node scripts/ensure-electron-installed.mjs", + "postinstall": "node scripts/ensure-electron-installed.mjs && node scripts/copy-framegen-weights.mjs", "electron:install": "node scripts/ensure-electron-installed.mjs", "dev": "electron-vite dev", "build": "electron-vite build", @@ -36,6 +36,7 @@ "dependencies": { "discord-rpc": "^4.0.1", "electron-updater": "^6.8.9", + "framegen": "^1.4.0", "posthog-node": "^5.48.1", "ws": "^8.21.3" }, @@ -50,6 +51,7 @@ "@types/three": "^0.185.4", "@types/ws": "^8.18.1", "@vitejs/plugin-react": "^5.2.0", + "@webgpu/types": "^0.1.71", "cross-env": "^10.1.0", "electron": "^43.3.0", "electron-builder": "^26.15.3", diff --git a/opennow-stable/scripts/copy-framegen-weights.mjs b/opennow-stable/scripts/copy-framegen-weights.mjs new file mode 100644 index 00000000..75b299a0 --- /dev/null +++ b/opennow-stable/scripts/copy-framegen-weights.mjs @@ -0,0 +1,23 @@ +/** + * Copy Framegen v7-small weights into the renderer public folder so the + * Electron app can load them offline from the app origin. + * + * Weight license is non-commercial (see third_party/framegen/WEIGHTS_LICENSE.md). + */ +import { cpSync, existsSync, mkdirSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = join(dirname(fileURLToPath(import.meta.url)), ".."); +const src = join(root, "node_modules", "framegen", "weights"); +const dest = join(root, "src", "renderer", "public", "framegen-weights"); + +if (!existsSync(src)) { + console.warn(`[copy-framegen-weights] skip: ${src} not found (install framegen first)`); + process.exit(0); +} + +mkdirSync(dest, { recursive: true }); +cpSync(src, dest, { recursive: true }); + +console.log(`[copy-framegen-weights] copied weights to ${dest}`); diff --git a/opennow-stable/src/main/settings.ts b/opennow-stable/src/main/settings.ts index 79affd32..b23b100c 100644 --- a/opennow-stable/src/main/settings.ts +++ b/opennow-stable/src/main/settings.ts @@ -18,6 +18,7 @@ import { normalizeStreamPreferences, normalizeTransportModeForPlatform, normalizeVideoShaderSettings, + normalizeFrameInterpolationSettings, normalizeUpdateChannel, normalizeRecordingBitrateMbps, normalizeRecordingFps, @@ -326,6 +327,12 @@ export class SettingsManager { migrated = true; } + const frameInterpolation = normalizeFrameInterpolationSettings(settings.frameInterpolation); + if (JSON.stringify(settings.frameInterpolation) !== JSON.stringify(frameInterpolation)) { + settings.frameInterpolation = frameInterpolation; + migrated = true; + } + const consentBefore = settings.errorReportingConsent; settings.errorReportingConsent = normalizeErrorReportingConsent(settings.errorReportingConsent); if (settings.errorReportingConsent !== consentBefore) { diff --git a/opennow-stable/src/renderer/public/.gitkeep b/opennow-stable/src/renderer/public/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/opennow-stable/src/renderer/src/App.tsx b/opennow-stable/src/renderer/src/App.tsx index 44b55952..6566a737 100644 --- a/opennow-stable/src/renderer/src/App.tsx +++ b/opennow-stable/src/renderer/src/App.tsx @@ -6,6 +6,7 @@ import type { ActiveSessionInfo, AuthSession, DirectLaunchRequest, + FrameInterpolationSettings, GameInfo, LoginProvider, NativeStreamerShortcutAction, @@ -1513,6 +1514,10 @@ export function App(): JSX.Element { void updateSetting("videoShader", value); }, [updateSetting]); + const handleFrameInterpolationChange = useCallback((value: FrameInterpolationSettings) => { + void updateSetting("frameInterpolation", value); + }, [updateSetting]); + const handleExitApp = useCallback(() => { appUnloadingRef.current = true; persistRuntimeSnapshotNow(); @@ -3044,6 +3049,8 @@ export function App(): JSX.Element { allowEscapeToExitFullscreen={settings.allowEscapeToExitFullscreen} videoShader={settings.videoShader} onVideoShaderChange={handleVideoShaderChange} + frameInterpolation={settings.frameInterpolation} + onFrameInterpolationChange={handleFrameInterpolationChange} /> )} diff --git a/opennow-stable/src/renderer/src/components/StreamView.tsx b/opennow-stable/src/renderer/src/components/StreamView.tsx index 14ed13bf..0ddf62ae 100644 --- a/opennow-stable/src/renderer/src/components/StreamView.tsx +++ b/opennow-stable/src/renderer/src/components/StreamView.tsx @@ -11,6 +11,7 @@ import { getStoreDisplayName, getStoreIconComponent } from "./GameCard"; import { SessionElapsedIndicator } from "./ElapsedSessionIndicators"; import { videoShaderHasVisibleEffect, + type FrameInterpolationSettings, type MicrophoneMode, type RecordingFps, type RecordingResolution, @@ -19,6 +20,7 @@ import { type VideoShaderSettings, } from "@shared/gfn"; import { VideoShaderPipeline } from "../platforms/gfn/videoShaderPipeline"; +import { FrameInterpolationPipeline } from "../platforms/gfn/frameInterpolationPipeline"; import { formatShortcutForDisplay } from "../shortcuts"; import { useScreenshotGallery } from "../hooks/useScreenshotGallery"; import { useStreamMenuNavigation } from "../hooks/useStreamMenuNavigation"; @@ -120,6 +122,8 @@ interface StreamViewProps { allowEscapeToExitFullscreen?: boolean; videoShader: VideoShaderSettings; onVideoShaderChange: (value: VideoShaderSettings) => void; + frameInterpolation: FrameInterpolationSettings; + onFrameInterpolationChange: (value: FrameInterpolationSettings) => void; } export function StreamView({ @@ -184,6 +188,8 @@ export function StreamView({ className, videoShader, onVideoShaderChange, + frameInterpolation, + onFrameInterpolationChange, }: StreamViewProps): JSX.Element { const [showHints, setShowHints] = useState(true); const [showSessionClock, setShowSessionClock] = useState(false); @@ -199,6 +205,7 @@ export function StreamView({ const localVideoRef = useRef(null); const localAudioRef = useRef(null); const shaderPipelineRef = useRef(null); + const frameInterpolationPipelineRef = useRef(null); const streamHasVideo = useStreamDiagnosticsSelector( diagnosticsStore, (stats) => hasVisibleStreamVideo(stats), @@ -447,9 +454,25 @@ export function StreamView({ } }, [videoShader, gstreamerEnabled, nativeRendererActive]); + useEffect(() => { + const video = localVideoRef.current; + if (!video) return; + const effective = gstreamerEnabled || nativeRendererActive + ? { ...frameInterpolation, enabled: false } + : frameInterpolation; + if (!frameInterpolationPipelineRef.current) { + if (!effective.enabled) return; + frameInterpolationPipelineRef.current = new FrameInterpolationPipeline(video, effective); + } else { + frameInterpolationPipelineRef.current.updateSettings(effective); + } + }, [frameInterpolation, gstreamerEnabled, nativeRendererActive]); + useEffect(() => () => { shaderPipelineRef.current?.dispose(); shaderPipelineRef.current = null; + frameInterpolationPipelineRef.current?.dispose(); + frameInterpolationPipelineRef.current = null; }, []); const setVideoRef = useCallback((element: HTMLVideoElement | null) => { @@ -762,6 +785,8 @@ export function StreamView({ gstreamerEnabled={gstreamerEnabled} videoShader={videoShader} onVideoShaderChange={onVideoShaderChange} + frameInterpolation={frameInterpolation} + onFrameInterpolationChange={onFrameInterpolationChange} microphoneMode={microphoneMode} onMicrophoneModeChange={onMicrophoneModeChange} diagnosticsStore={diagnosticsStore} diff --git a/opennow-stable/src/renderer/src/components/settings/settingsTypes.ts b/opennow-stable/src/renderer/src/components/settings/settingsTypes.ts index bfe64b28..9c05dc9d 100644 --- a/opennow-stable/src/renderer/src/components/settings/settingsTypes.ts +++ b/opennow-stable/src/renderer/src/components/settings/settingsTypes.ts @@ -95,6 +95,9 @@ export const SETTINGS_SCOPE_SEARCH_TERMS: Record +
+ + +
+ + {nativeBlocked + ? t("settings.frameInterpolation.nativeUnavailable") + : t("settings.frameInterpolation.hint")} + + {fi.enabled && !nativeBlocked && ( + <> +
+
+ + {t("settings.frameInterpolation.factor")} + + {fi.factor}× +
+
+ {FRAME_INTERPOLATION_FACTOR_OPTIONS.map((factor) => ( + + ))} +
+
+
+
+ + {t("settings.frameInterpolation.quality")} + + {fi.quality}p +
+
+ {FRAME_INTERPOLATION_QUALITY_OPTIONS.map((quality) => ( + + ))} +
+ + {t("settings.frameInterpolation.qualityHint")} + +
+

+ {t("settings.frameInterpolation.weightsNotice")} +

+
+ +
+ + )} + + ); +} diff --git a/opennow-stable/src/renderer/src/components/settings/stream/StreamVideoSection.tsx b/opennow-stable/src/renderer/src/components/settings/stream/StreamVideoSection.tsx index 1e146a01..8dae2386 100644 --- a/opennow-stable/src/renderer/src/components/settings/stream/StreamVideoSection.tsx +++ b/opennow-stable/src/renderer/src/components/settings/stream/StreamVideoSection.tsx @@ -7,6 +7,7 @@ import type { SettingsChangeHandler } from "./streamSettingsTypes"; import { StreamQualityControls } from "./StreamQualityControls"; import { SessionProxySettings } from "./SessionProxySettings"; import { VideoShaderControls } from "./VideoShaderControls"; +import { FrameInterpolationControls } from "./FrameInterpolationControls"; import { StreamSessionOptions } from "./StreamSessionOptions"; interface StreamVideoSectionProps { @@ -87,6 +88,10 @@ export function StreamVideoSection({ handleChange={handleChange} handlePreview={handlePreview} /> + diff --git a/opennow-stable/src/renderer/src/components/stream/quick-menu/StreamQuickMenu.tsx b/opennow-stable/src/renderer/src/components/stream/quick-menu/StreamQuickMenu.tsx index 8ab8348f..0854725d 100644 --- a/opennow-stable/src/renderer/src/components/stream/quick-menu/StreamQuickMenu.tsx +++ b/opennow-stable/src/renderer/src/components/stream/quick-menu/StreamQuickMenu.tsx @@ -3,6 +3,7 @@ import type { Dispatch, JSX, RefObject, SetStateAction } from "react"; import { AnimatePresence, m } from "motion/react"; import { Bug, Gauge, Images, Keyboard, LogOut, Save, SlidersHorizontal, Trash2, X } from "lucide-react"; import type { + FrameInterpolationSettings, MicrophoneMode, RecordingFps, RecordingResolution, @@ -55,6 +56,8 @@ interface StreamQuickMenuProps { gstreamerEnabled: boolean; videoShader: VideoShaderSettings; onVideoShaderChange: (value: VideoShaderSettings) => void; + frameInterpolation: FrameInterpolationSettings; + onFrameInterpolationChange: (value: FrameInterpolationSettings) => void; microphoneMode: MicrophoneMode; onMicrophoneModeChange: (value: MicrophoneMode) => void; diagnosticsStore: StreamDiagnosticsStore; @@ -104,6 +107,8 @@ export function StreamQuickMenu({ gstreamerEnabled, videoShader, onVideoShaderChange, + frameInterpolation, + onFrameInterpolationChange, microphoneMode, onMicrophoneModeChange, diagnosticsStore, @@ -256,6 +261,8 @@ export function StreamQuickMenu({ gstreamerEnabled={gstreamerEnabled} videoShader={videoShader} onVideoShaderChange={onVideoShaderChange} + frameInterpolation={frameInterpolation} + onFrameInterpolationChange={onFrameInterpolationChange} microphoneMode={microphoneMode} onMicrophoneModeChange={onMicrophoneModeChange} diagnosticsStore={diagnosticsStore} diff --git a/opennow-stable/src/renderer/src/components/stream/quick-menu/StreamQuickMenuControlsPage.tsx b/opennow-stable/src/renderer/src/components/stream/quick-menu/StreamQuickMenuControlsPage.tsx index ed47413a..925c2fea 100644 --- a/opennow-stable/src/renderer/src/components/stream/quick-menu/StreamQuickMenuControlsPage.tsx +++ b/opennow-stable/src/renderer/src/components/stream/quick-menu/StreamQuickMenuControlsPage.tsx @@ -1,6 +1,14 @@ import type { JSX, RefObject } from "react"; -import type { MicrophoneMode, VideoShaderSettings } from "@shared/gfn"; -import { DEFAULT_VIDEO_SHADER_SETTINGS } from "@shared/gfn"; +import type { + FrameInterpolationSettings, + MicrophoneMode, + VideoShaderSettings, +} from "@shared/gfn"; +import { + DEFAULT_VIDEO_SHADER_SETTINGS, + FRAME_INTERPOLATION_FACTOR_OPTIONS, + FRAME_INTERPOLATION_QUALITY_OPTIONS, +} from "@shared/gfn"; import type { StreamDiagnosticsStore } from "../../../utils/streamDiagnosticsStore"; import { SidebarMicMutedBadge } from "../StreamEmptyStates"; @@ -27,6 +35,8 @@ interface StreamQuickMenuControlsPageProps { gstreamerEnabled: boolean; videoShader: VideoShaderSettings; onVideoShaderChange: (value: VideoShaderSettings) => void; + frameInterpolation: FrameInterpolationSettings; + onFrameInterpolationChange: (value: FrameInterpolationSettings) => void; microphoneMode: MicrophoneMode; onMicrophoneModeChange: (value: MicrophoneMode) => void; diagnosticsStore: StreamDiagnosticsStore; @@ -42,6 +52,8 @@ export function StreamQuickMenuControlsPage({ gstreamerEnabled, videoShader, onVideoShaderChange, + frameInterpolation, + onFrameInterpolationChange, microphoneMode, onMicrophoneModeChange, diagnosticsStore, @@ -103,6 +115,82 @@ export function StreamQuickMenuControlsPage({