Skip to content

Fix timeline gap frame conditioning #134

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Paul (nuclearspike) wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Fix timeline gap frame conditioning #134

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 52 additions & 10 deletions frontend/views/editor/GapGenerationModal.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,16 @@ interface TimelineGap {

type GapGenerateMode = 'text-to-video' | 'image-to-video' | 'text-to-image'

export interface GapFrameConditioning {
imagePath: string | null
imageFile: File | null
reverseResult: boolean
}

interface FrameOverride {
file: File
previewUrl: string
}

interface GapGenerationModalProps {
selectedGap: TimelineGap | null
Expand All @@ -41,7 +51,7 @@ interface GapGenerationModalProps {
regenStatusMessage: string
regenProgress: number
regenReset: () => void
handleGapGenerate: () => void
handleGapGenerate: (conditioning: GapFrameConditioning) => void
handleCloseGap: () => void
setSelectedGap: (gap: TimelineGap | null) => void
gapApplyAudioToTrack: boolean
Expand Down Expand Up @@ -119,8 +129,8 @@ export function GapGenerationModal({

const [startFrameEnabled, setStartFrameEnabled] = useState(true)
const [endFrameEnabled, setEndFrameEnabled] = useState(false)
const [startFrameOverride, setStartFrameOverride] = useState<string | null>(null)
const [endFrameOverride, setEndFrameOverride] = useState<string | null>(null)
const [startFrameOverride, setStartFrameOverride] = useState<FrameOverride | null>(null)
const [endFrameOverride, setEndFrameOverride] = useState<FrameOverride | null>(null)
const startFrameInputRef = useRef<HTMLInputElement>(null)
const endFrameInputRef = useRef<HTMLInputElement>(null)

Expand All @@ -131,22 +141,54 @@ export function GapGenerationModal({
setEndFrameOverride(null)
}, [gapGenerateMode])

const displayedBeforeFrame = startFrameOverride ?? gapBeforeFrame
const displayedAfterFrame = endFrameOverride ?? gapAfterFrame
const displayedBeforeFrame = startFrameOverride?.previewUrl
?? (gapBeforeFrame ? pathToFileUrl(gapBeforeFrame) : null)
const displayedAfterFrame = endFrameOverride?.previewUrl
?? (gapAfterFrame ? pathToFileUrl(gapAfterFrame) : null)

const handleFrameFileChange = (
e: React.ChangeEvent<HTMLInputElement>,
setter: (v: string | null) => void,
setter: (v: FrameOverride | null) => void,
onSelect: () => void
) => {
const file = e.target.files?.[0]
if (!file) return
const reader = new FileReader()
reader.onload = (ev) => { setter(ev.target?.result as string); onSelect() }
reader.onload = (ev) => {
setter({ file, previewUrl: ev.target?.result as string })
onSelect()
}
reader.readAsDataURL(file)
e.target.value = ''
}

const handleGenerate = () => {
if (!isVideoMode || gapImageFile) {
handleGapGenerate({ imagePath: null, imageFile: null, reverseResult: false })
return
}

if (startFrameEnabled && (startFrameOverride || gapBeforeFrame)) {
handleGapGenerate({
imagePath: startFrameOverride ? null : gapBeforeFrame,
imageFile: startFrameOverride?.file ?? null,
reverseResult: false,
})
return
}

if (endFrameEnabled && (endFrameOverride || gapAfterFrame)) {
handleGapGenerate({
imagePath: endFrameOverride ? null : gapAfterFrame,
imageFile: endFrameOverride?.file ?? null,
reverseResult: true,
})
return
}

handleGapGenerate({ imagePath: null, imageFile: null, reverseResult: false })
}

return (
<>
{gapGenerateMode && (
Expand Down Expand Up @@ -229,7 +271,7 @@ export function GapGenerationModal({
onClick={() => { if (startFrameEnabled) { setStartFrameEnabled(false) } else { setStartFrameEnabled(true); setEndFrameEnabled(false) } }}
>
<img
src={pathToFileUrl(displayedBeforeFrame)}
src={displayedBeforeFrame}
alt=""
className={`w-full h-full object-cover transition-all duration-300 ${
!startFrameEnabled ? 'grayscale opacity-50' : ''
Expand Down Expand Up @@ -295,7 +337,7 @@ export function GapGenerationModal({
onClick={() => { if (endFrameEnabled) { setEndFrameEnabled(false) } else { setEndFrameEnabled(true); setStartFrameEnabled(false) } }}
>
<img
src={pathToFileUrl(displayedAfterFrame)}
src={displayedAfterFrame}
alt=""
className={`w-full h-full object-cover transition-all duration-300 ${
!endFrameEnabled ? 'grayscale opacity-50' : ''
Expand Down Expand Up @@ -480,7 +522,7 @@ export function GapGenerationModal({
Cancel
</button>
<button
onClick={handleGapGenerate}
onClick={handleGenerate}
disabled={isRegenerating || !gapPrompt.trim() || (isVideoMode && !gapCanGenerateVideo)}
className="px-4 py-1.5 rounded-md bg-blue-600 text-white text-sm hover:bg-blue-500 transition-colors font-medium disabled:opacity-40 disabled:cursor-not-allowed flex items-center gap-1.5"
>
Expand Down
39 changes: 23 additions & 16 deletions frontend/views/editor/VideoEditorTimelineEditingPanel.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import { useAppSettings } from '../../contexts/AppSettingsContext'
import { useVideoGenerationModelSpecs } from '../../hooks/use-video-generation-model-specs'
import type { GenerationError } from '../../lib/generation-errors'
import { addVisualAssetToProject } from '../../lib/asset-copy'
import { GapGenerationModal } from './GapGenerationModal'
import { GapGenerationModal, type GapFrameConditioning } from './GapGenerationModal'
import { ClipContextMenu, type ClipContextMenuState } from './ClipContextMenu'
import type { TimelineClip, Track, SubtitleClip, Asset, TextOverlayStyle } from '../../types/project-model'
import { ApiClient } from '../../lib/api-client'
Expand Down Expand Up @@ -105,6 +105,20 @@ interface GapGenerationApi {
error: GenerationError | null
}

async function materializeGapImageFile(file: File): Promise<string> {
const electronPath = window.electronAPI.getPathForFile(file)
if (electronPath) return electronPath

// Retain the byte-save fallback for browser-like File objects without a native path.
const buf = await file.arrayBuffer()
const b64 = btoa(String.fromCharCode(...new Uint8Array(buf)))
const modelsPath = await window.electronAPI.getModelsPath()
const tmpDir = modelsPath.replace(/[/\\]models$/, '')
const tmpPath = `${tmpDir}/tmp_gap_image_${Date.now()}.png`
await window.electronAPI.saveFile({ filePath: tmpPath, data: b64, encoding: 'base64' })
return tmpPath
}

export interface VideoEditorTimelineEditingPanelProps {
currentProjectId: string | null
playbackTimeRef: React.MutableRefObject<number>
Expand Down Expand Up @@ -435,6 +449,7 @@ export function VideoEditorTimelineEditingPanel(props: VideoEditorTimelineEditin
prompt: string
settings: GenerationSettings
applyAudio: boolean
reverseResult: boolean
} | null>(null)
const [gapSuggesting, setGapSuggesting] = useState(false)
const [gapSuggestion, setGapSuggestion] = useState<string | null>(null)
Expand Down Expand Up @@ -850,7 +865,7 @@ export function VideoEditorTimelineEditingPanel(props: VideoEditorTimelineEditin
runSuggestion(true)
}, [runSuggestion])

const handleGapGenerate = useCallback(async () => {
const handleGapGenerate = useCallback(async (frameConditioning: GapFrameConditioning) => {
if (!selectedGap || !gapGenerateMode || !gapPrompt.trim() || !currentProjectId) return

const gap = selectedGap
Expand All @@ -872,6 +887,7 @@ export function VideoEditorTimelineEditingPanel(props: VideoEditorTimelineEditin
prompt: finalPrompt,
settings,
applyAudio: gapApplyAudioToTrack,
reverseResult: mode !== 'text-to-image' && !gapImageFile && frameConditioning.reverseResult,
})

clearSelectedGap()
Expand All @@ -880,20 +896,10 @@ export function VideoEditorTimelineEditingPanel(props: VideoEditorTimelineEditin
if (mode === 'text-to-image') {
await gapGenerationApi.generateImage(finalPrompt, settings)
} else {
let imagePath: string | null = null
if (gapImageFile) {
const electronPath = (gapImageFile as { path?: string }).path
if (electronPath) {
imagePath = electronPath
} else {
const buf = await gapImageFile.arrayBuffer()
const b64 = btoa(String.fromCharCode(...new Uint8Array(buf)))
const modelsPath = await window.electronAPI.getModelsPath()
const tmpDir = modelsPath.replace(/[/\\]models$/, '')
const tmpPath = `${tmpDir}/tmp_gap_image_${Date.now()}.png`
await window.electronAPI.saveFile({ filePath: tmpPath, data: b64, encoding: 'base64' })
imagePath = tmpPath
}
let imagePath = frameConditioning.imagePath
const imageFile = gapImageFile ?? frameConditioning.imageFile
if (imageFile) {
imagePath = await materializeGapImageFile(imageFile)
}
await gapGenerationApi.generate(finalPrompt, imagePath, settings)
}
Expand Down Expand Up @@ -974,6 +980,7 @@ export function VideoEditorTimelineEditingPanel(props: VideoEditorTimelineEditin
gap,
asset,
createAudio: assetType === 'video' && generatingGap.applyAudio && generatingGap.settings.audio,
reversed: assetType === 'video' && generatingGap.reverseResult,
})
}

Expand Down
5 changes: 3 additions & 2 deletions frontend/views/editor/editor-actions.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ export interface InsertGeneratedGapAssetParams {
gap: TimelineGapSelection
asset: Asset
createAudio: boolean
reversed: boolean
}

export interface SourceEditParams {
Expand Down Expand Up @@ -1579,7 +1580,7 @@ export function insertGeneratedGapAsset(state: EditorState, params: InsertGenera
trimStart: 0,
trimEnd: 0,
speed: 1,
reversed: false,
reversed: params.reversed,
muted: false,
volume: 1,
trackIndex: params.gap.trackIndex,
Expand All @@ -1603,7 +1604,7 @@ export function insertGeneratedGapAsset(state: EditorState, params: InsertGenera
trimStart: 0,
trimEnd: 0,
speed: 1,
reversed: false,
reversed: params.reversed,
muted: false,
volume: 1,
trackIndex: audioTrackIndex,
Expand Down