diff --git a/backend/_routes/health.py b/backend/_routes/health.py index d66789cf5..757b7357c 100644 --- a/backend/_routes/health.py +++ b/backend/_routes/health.py @@ -7,7 +7,7 @@ from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request -from api_types import GpuInfoResponse, HealthResponse, MpsMemoryResponse +from api_types import GpuInfoResponse, HealthResponse, MpsMemoryResponse, RuntimeTelemetryResponse from state import get_state_service from app_handler import AppHandler @@ -29,6 +29,11 @@ def route_mps_memory(handler: AppHandler = Depends(get_state_service)) -> MpsMem return handler.health.get_mps_memory() +@router.get("/api/runtime-telemetry", response_model=RuntimeTelemetryResponse) +def route_runtime_telemetry(handler: AppHandler = Depends(get_state_service)) -> RuntimeTelemetryResponse: + return handler.health.get_runtime_telemetry() + + def _shutdown_process() -> None: os.kill(os.getpid(), signal.SIGTERM) diff --git a/backend/api_types.py b/backend/api_types.py index 7defc671e..f2c09368c 100644 --- a/backend/api_types.py +++ b/backend/api_types.py @@ -90,8 +90,52 @@ class MpsMemoryResponse(BaseModel): recommended_max_mib: int | None = None +RuntimeEngine: TypeAlias = Literal["torch", "mlx", "cloud"] + + +class RuntimeProvenanceItem(BaseModel): + component: str + version: str + revision: str | None = None + source: str + + class RuntimePolicyResponse(BaseModel): force_api_generations: bool + fast_video_engine_preference: Literal["auto", "torch", "mlx"] + auto_fast_video_engine: Literal["torch", "mlx", "cloud"] + auto_selection_reason: str + execution_mode: Literal["eager", "low_ram", "unsupported"] + automatic_tiling: bool + mlx_model_source: str + mlx_model_variant: Literal["bf16", "q8"] + quality_warning: str | None = None + capability_engines: dict[str, RuntimeEngine] + provenance: list[RuntimeProvenanceItem] + + +class RuntimeTelemetryResponse(BaseModel): + sampled_at: str + active_engine: RuntimeEngine | None = None + active_pipeline: str | None = None + process_rss_mib: int + system_total_mib: int + system_available_mib: int + mlx_active_mib: int | None = None + mlx_cache_mib: int | None = None + mlx_peak_mib: int | None = None + mlx_profile_status: Literal["running", "success", "error", "cancelled"] | None = None + mlx_profile_phase: str | None = None + mlx_profile_path: str | None = None + mlx_profile_sampled_at: str | None = None + mlx_runtime_identity: dict[str, object] | None = None + mps_allocated_mib: int | None = None + mps_driver_mib: int | None = None + mps_recommended_max_mib: int | None = None + local_metal_lease_status: Literal["idle", "waiting", "held"] + local_metal_lease_reason: str | None = None + local_metal_lease_waited_seconds: float = 0.0 + local_metal_lease_owner: dict[str, object] | None = None class GenerationProgressResponse(BaseModel): @@ -143,6 +187,8 @@ class SuggestGapPromptResponse(BaseModel): class GenerateVideoCompleteResponse(BaseModel): status: Literal["complete"] video_path: str + resolved_width: int | None = None + resolved_height: int | None = None class GenerateVideoCancelledResponse(BaseModel): diff --git a/backend/app_handler.py b/backend/app_handler.py index f9e1647eb..234cf92ee 100644 --- a/backend/app_handler.py +++ b/backend/app_handler.py @@ -68,6 +68,7 @@ def __init__( ltx_api_client: LTXAPIClient, zit_api_client: ZitAPIClient, fast_video_pipeline_class: type[FastVideoPipeline], + mlx_fast_video_pipeline_class: type[FastVideoPipeline], image_generation_pipeline_class: type[ImageGenerationPipeline], ic_lora_pipeline_class: type[IcLoraPipeline], depth_processor_pipeline_class: type[DepthProcessorPipeline], @@ -88,6 +89,7 @@ def __init__( self.ltx_api_client = ltx_api_client self.zit_api_client = zit_api_client self.fast_video_pipeline_class = fast_video_pipeline_class + self.mlx_fast_video_pipeline_class = mlx_fast_video_pipeline_class self.image_generation_pipeline_class = image_generation_pipeline_class self.ic_lora_pipeline_class = ic_lora_pipeline_class self.depth_processor_pipeline_class = depth_processor_pipeline_class @@ -160,6 +162,7 @@ def __init__( text_handler=self.text, gpu_cleaner=gpu_cleaner, fast_video_pipeline_class=fast_video_pipeline_class, + mlx_fast_video_pipeline_class=mlx_fast_video_pipeline_class, image_generation_pipeline_class=image_generation_pipeline_class, ic_lora_pipeline_class=ic_lora_pipeline_class, depth_processor_pipeline_class=depth_processor_pipeline_class, @@ -273,6 +276,7 @@ class ServiceBundle: ltx_api_client: LTXAPIClient zit_api_client: ZitAPIClient fast_video_pipeline_class: type[FastVideoPipeline] + mlx_fast_video_pipeline_class: type[FastVideoPipeline] image_generation_pipeline_class: type[ImageGenerationPipeline] ic_lora_pipeline_class: type[IcLoraPipeline] depth_processor_pipeline_class: type[DepthProcessorPipeline] @@ -285,6 +289,7 @@ class ServiceBundle: def build_default_service_bundle(config: RuntimeConfig) -> ServiceBundle: """Build real runtime services with lazy heavy imports isolated from tests.""" from services.fast_video_pipeline.ltx_fast_video_pipeline import LTXFastVideoPipeline + from services.fast_video_pipeline.mlx_fast_video_pipeline import MLXFastVideoPipeline from services.zit_api_client.zit_api_client_impl import ZitAPIClientImpl from services.gpu_cleaner.torch_cleaner import TorchCleaner from services.gpu_info.gpu_info_impl import GpuInfoImpl @@ -323,6 +328,7 @@ def build_default_service_bundle(config: RuntimeConfig) -> ServiceBundle: ltx_api_client=LTXAPIClientImpl(http=http, ltx_api_base_url=config.ltx_api_base_url), zit_api_client=ZitAPIClientImpl(http=http), fast_video_pipeline_class=LTXFastVideoPipeline, + mlx_fast_video_pipeline_class=MLXFastVideoPipeline, image_generation_pipeline_class=ZitImageGenerationPipeline, ic_lora_pipeline_class=LTXIcLoraPipeline, depth_processor_pipeline_class=MidasDPTPipeline, @@ -354,6 +360,7 @@ def build_initial_state( ltx_api_client=bundle.ltx_api_client, zit_api_client=bundle.zit_api_client, fast_video_pipeline_class=bundle.fast_video_pipeline_class, + mlx_fast_video_pipeline_class=bundle.mlx_fast_video_pipeline_class, image_generation_pipeline_class=bundle.image_generation_pipeline_class, ic_lora_pipeline_class=bundle.ic_lora_pipeline_class, depth_processor_pipeline_class=bundle.depth_processor_pipeline_class, diff --git a/backend/export_openapi_schema.py b/backend/export_openapi_schema.py index 3cadbc2ad..f746e362e 100644 --- a/backend/export_openapi_schema.py +++ b/backend/export_openapi_schema.py @@ -64,6 +64,7 @@ def _build_schema() -> dict[str, object]: ltx_api_client=cast(Any, fake.ltx_api_client), zit_api_client=cast(Any, fake.zit_api_client), fast_video_pipeline_class=cast(Any, type(fake.fast_video_pipeline)), + mlx_fast_video_pipeline_class=cast(Any, type(fake.fast_video_pipeline)), image_generation_pipeline_class=cast(Any, type(fake.image_generation_pipeline)), ic_lora_pipeline_class=cast(Any, type(fake.ic_lora_pipeline)), depth_processor_pipeline_class=cast(Any, type(fake.depth_processor_pipeline)), diff --git a/backend/handlers/extend_handler.py b/backend/handlers/extend_handler.py index 22c67be50..a3888c6b6 100644 --- a/backend/handlers/extend_handler.py +++ b/backend/handlers/extend_handler.py @@ -164,16 +164,21 @@ def _run_local_extend( extend_frames = self._duration_to_extend_frames(duration, fps) target_width, target_height = resolve_target_resolution(resolution, source_width, source_height) - try: - self._text.prepare_text_encoding(prompt, enhance_prompt=False) - except RuntimeError as exc: - raise HTTPError(400, str(exc)) from exc - generation_id = uuid.uuid4().hex[:8] seed = self._resolve_seed() output_path = self.config.outputs_dir / f"extend_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{generation_id}.mp4" + lease = None try: + lease = self._generation.acquire_local_metal_lease( + generation_id=generation_id, + workload="extend", + reason="Torch local extend generation", + ) + try: + self._text.prepare_text_encoding(prompt, enhance_prompt=False) + except RuntimeError as exc: + raise HTTPError(400, str(exc)) from exc pipeline_state = self._pipelines.load_retake_pipeline(distilled=True) self._generation.start_generation(generation_id) self._generation.update_progress("loading_model", 5, 0, 1) @@ -212,6 +217,9 @@ def _run_local_extend( raise HTTPError(500, f"Generation error: {exc}") from exc finally: self._text.clear_api_embeddings() + if lease is not None: + self._pipelines.cleanup_runtime_caches() + lease.close() @staticmethod def _duration_to_extend_frames(duration: float, fps: float) -> int: diff --git a/backend/handlers/generation_handler.py b/backend/handlers/generation_handler.py index 123092c13..0898ad74c 100644 --- a/backend/handlers/generation_handler.py +++ b/backend/handlers/generation_handler.py @@ -17,7 +17,9 @@ GenerationProgressResponse, ) from handlers.base import StateHandlerBase, with_state_lock +from services.fast_video_pipeline.mlx_fast_video_pipeline import cancel_active_mlx_sidecar from services.patches import diffusion_stage_cache +from services.local_metal_lease import LocalMetalLeaseHandle, local_metal_lease from state.app_state_types import ( ApiGeneration, AppState, @@ -63,11 +65,29 @@ def try_reserve_generation_start(self) -> bool: logger.info("Generation start reservation denied: another reservation is still active") return False self.state.generation_starting_since = time.monotonic() + self.state.generation_starting_id = None + self.state.generation_starting_phase = "starting" + self.state.generation_start_cancelled = False return True @with_state_lock def release_generation_start_reservation(self) -> None: self.state.generation_starting_since = None + self.state.generation_starting_id = None + self.state.generation_starting_phase = "starting" + self.state.generation_start_cancelled = False + + @with_state_lock + def annotate_generation_start(self, generation_id: str, phase: str) -> None: + if self.state.generation_starting_since is None: + return + self.state.generation_starting_id = generation_id + self.state.generation_starting_phase = phase + + @with_state_lock + def update_generation_start_phase(self, phase: str) -> None: + if self.state.generation_starting_since is not None: + self.state.generation_starting_phase = phase @contextmanager def reserved_generation_start(self) -> Iterator[None]: @@ -88,19 +108,82 @@ def reserved_generation_start(self) -> Iterator[None]: finally: self.release_generation_start_reservation() + @contextmanager + def hold_local_metal_lease( + self, + *, + generation_id: str, + workload: str, + reason: str, + ) -> Iterator[None]: + """Hold the cross-product Metal lease before any local model load.""" + self.annotate_generation_start(generation_id, "waiting_for_local_accelerator") + + def _on_wait(waited: float, owner: dict[str, object] | None) -> None: + owner_product = owner.get("product") if owner else "another LTX app" + owner_pid = owner.get("pid") if owner else "?" + self.update_generation_start_phase( + f"waiting_for_local_accelerator:{owner_product}:pid={owner_pid}:" + f"{waited:.1f}s" + ) + + with local_metal_lease( + job_id=generation_id, + workload=workload, + reason=reason, + is_cancelled=self.is_generation_cancelled, + on_wait=_on_wait, + ): + self.update_generation_start_phase("loading_model") + yield + + def acquire_local_metal_lease( + self, + *, + generation_id: str, + workload: str, + reason: str, + ) -> LocalMetalLeaseHandle: + """Acquire an explicit lease handle; caller closes it after GPU cleanup.""" + self.annotate_generation_start(generation_id, "waiting_for_local_accelerator") + + def _on_wait(waited: float, owner: dict[str, object] | None) -> None: + owner_product = owner.get("product") if owner else "another LTX app" + owner_pid = owner.get("pid") if owner else "?" + self.update_generation_start_phase( + f"waiting_for_local_accelerator:{owner_product}:pid={owner_pid}:" + f"{waited:.1f}s" + ) + + handle = LocalMetalLeaseHandle( + local_metal_lease( + job_id=generation_id, + workload=workload, + reason=reason, + is_cancelled=self.is_generation_cancelled, + on_wait=_on_wait, + ) + ) + self.update_generation_start_phase("loading_model") + return handle + @with_state_lock def start_generation(self, generation_id: str) -> None: if self.is_generation_running(): raise RuntimeError("Generation already in progress") + if self.state.generation_start_cancelled: + raise RuntimeError("Generation was cancelled before model loading completed") if self.state.gpu_slot is None: raise RuntimeError("No active GPU pipeline") self.state.generation_starting_since = None - - # EXPERIMENTAL: push the live Settings toggle, then drop any transformer - # cached from the previous generation before this one starts -- otherwise - # it stays resident while this generation's own text encoder/VAE/etc. - # build, double-booking VRAM. See that module's GENERATION-SCOPED - # docstring section for the RTX 5090 repro (~42GB reported on a 32GB card). + self.state.generation_starting_id = None + self.state.generation_starting_phase = "starting" + self.state.generation_start_cancelled = False + + # Push the live Settings toggle, then drop any transformer left by a + # cancelled/failed generation before this one starts. A normal two-stage + # generation retires the cache immediately after the reuse hit, before VAE + # decode; this is the defensive backstop. See diffusion_stage_cache.py. diffusion_stage_cache.set_enabled(self.state.app_settings.diffusion_stage_cache_enabled) diffusion_stage_cache.evict() @@ -210,6 +293,8 @@ def _generation_for_polling(self) -> GenerationState | None: @with_state_lock def is_generation_cancelled(self) -> bool: + if self.state.generation_start_cancelled: + return True match self._active_generation_state(): case (_, GenerationCancelled()): return True @@ -236,10 +321,18 @@ def update_progress( @with_state_lock def cancel_generation(self) -> CancelResponse: + if self.state.generation_starting_since is not None: + self.state.generation_start_cancelled = True + generation_id = self.state.generation_starting_id or "starting" + return CancelCancellingResponse(status="cancelling", id=generation_id) running_generation = self._running_generation() if running_generation is not None: slot, running = running_generation self._set_generation_state(slot, GenerationCancelled(id=running.id)) + # Torch pipelines observe the state cooperatively. MLX is isolated + # in a fresh child process, so cancellation tears down its whole + # process group and releases unified memory deterministically. + cancel_active_mlx_sidecar() return CancelCancellingResponse(status="cancelling", id=running.id) cancelled_generation = self._cancelled_generation() @@ -265,6 +358,9 @@ def fail_generation(self, error: str) -> None: # happened before start_generation()/start_api_generation() ever ran (e.g. pipeline load # itself threw) — that path never touches generation_starting_since otherwise. self.state.generation_starting_since = None + self.state.generation_starting_id = None + self.state.generation_starting_phase = "starting" + self.state.generation_start_cancelled = False running_generation = self._running_generation() if running_generation is not None: slot, running = running_generation @@ -286,6 +382,16 @@ def get_generation_progress(self) -> GenerationProgressResponse: # until start_generation() overwrites it. Matching on gen first would report that stale # terminal state instead of "starting" for every generation after the first. if self.state.generation_starting_since is not None: + generation_id = self.state.generation_starting_id + if self.state.generation_start_cancelled: + return GenerationProgressResponse( + status="cancelled", + phase="cancelled", + progress=0, + currentStep=0, + totalSteps=0, + id=generation_id, + ) # Reserved (try_reserve_generation_start succeeded) but pipeline load hasn't # finished, so start_generation() hasn't run and there's no real id yet. # Still report "running": a client polling for "is anything busy right now" @@ -295,10 +401,11 @@ def get_generation_progress(self) -> GenerationProgressResponse: # docstring for why this window needed closing on the write side too. return GenerationProgressResponse( status="running", - phase="starting", + phase=self.state.generation_starting_phase, progress=0, currentStep=0, totalSteps=0, + id=generation_id, ) gen = self._generation_for_polling() diff --git a/backend/handlers/health_handler.py b/backend/handlers/health_handler.py index a20083dfe..41fe59d41 100644 --- a/backend/handlers/health_handler.py +++ b/backend/handlers/health_handler.py @@ -2,13 +2,26 @@ from __future__ import annotations +from datetime import UTC, datetime +import os from threading import RLock from typing import TYPE_CHECKING -from api_types import GpuInfoResponse, GpuTelemetry, HealthResponse, ModelStatusItem, MpsMemoryResponse +import psutil + +from api_types import ( + GpuInfoResponse, + GpuTelemetry, + HealthResponse, + ModelStatusItem, + MpsMemoryResponse, + RuntimeTelemetryResponse, +) from handlers.base import StateHandlerBase from handlers.models_handler import ModelsHandler +from services.fast_video_pipeline.mlx_profile import get_mlx_profile_snapshot from services.interfaces import GpuInfo +from services.local_metal_lease import get_local_metal_lease_snapshot from state.app_state_types import AppState, GpuSlot, VideoPipelineState if TYPE_CHECKING: @@ -89,3 +102,48 @@ def get_mps_memory(self) -> MpsMemoryResponse: ) except Exception: # noqa: BLE001 return MpsMemoryResponse(available=False) + + def get_runtime_telemetry(self) -> RuntimeTelemetryResponse: + """Return one cheap process/system/accelerator memory sample.""" + process_rss_mib = round(psutil.Process(os.getpid()).memory_info().rss / _BYTES_PER_MIB) + virtual_memory = psutil.virtual_memory() + + active_engine = None + active_pipeline = None + with self._lock: + if self.state.gpu_slot is not None: + active = self.state.gpu_slot.active_pipeline + if isinstance(active, VideoPipelineState): + active_engine = active.runtime_engine + active_pipeline = active.pipeline.pipeline_kind + else: + active_engine = "torch" + active_pipeline = type(active).__name__ + + mlx_profile = get_mlx_profile_snapshot() + + mps = self.get_mps_memory() + lease = get_local_metal_lease_snapshot() + return RuntimeTelemetryResponse( + sampled_at=datetime.now(UTC).isoformat(), + active_engine=active_engine, + active_pipeline=active_pipeline, + process_rss_mib=process_rss_mib, + system_total_mib=round(virtual_memory.total / _BYTES_PER_MIB), + system_available_mib=round(virtual_memory.available / _BYTES_PER_MIB), + mlx_active_mib=mlx_profile.active_mib if mlx_profile else None, + mlx_cache_mib=mlx_profile.cache_mib if mlx_profile else None, + mlx_peak_mib=mlx_profile.peak_mib if mlx_profile else None, + mlx_profile_status=mlx_profile.status if mlx_profile else None, + mlx_profile_phase=mlx_profile.phase if mlx_profile else None, + mlx_profile_path=mlx_profile.profile_path if mlx_profile else None, + mlx_profile_sampled_at=mlx_profile.sampled_at if mlx_profile else None, + mlx_runtime_identity=mlx_profile.runtime_identity if mlx_profile else None, + mps_allocated_mib=mps.allocated_mib, + mps_driver_mib=mps.driver_mib, + mps_recommended_max_mib=mps.recommended_max_mib, + local_metal_lease_status=lease["status"], + local_metal_lease_reason=lease["reason"], + local_metal_lease_waited_seconds=lease["waited_seconds"], + local_metal_lease_owner=lease["owner"], + ) diff --git a/backend/handlers/ic_lora_handler.py b/backend/handlers/ic_lora_handler.py index f09191df4..0b36afc88 100644 --- a/backend/handlers/ic_lora_handler.py +++ b/backend/handlers/ic_lora_handler.py @@ -284,7 +284,13 @@ def _generate_ic_lora(self, req: IcLoraGenerateRequest) -> IcLoraGenerateRespons s = _resolve_settings(req, ic_lora.default_settings) generation_id = uuid.uuid4().hex[:8] logger.info("[ic-lora] IC-LoRA generation started (ic_lora=%s)", ic_lora.id) + lease = None try: + lease = self._generation.acquire_local_metal_lease( + generation_id=generation_id, + workload=f"ic_lora:{ic_lora.id}", + reason="Torch local IC-LoRA generation", + ) # IC-LoRA preprocessing builds the control video itself; no depth processor needed. ic_state = self._pipelines.load_ic_lora(str(lora_path), None, s.lora_strength) self._generation.start_generation(generation_id) @@ -403,6 +409,9 @@ def _generate_ic_lora(self, req: IcLoraGenerateRequest) -> IcLoraGenerateRespons raise HTTPError(500, f"Generation error: {exc}") from exc finally: self._text.clear_api_embeddings() + if lease is not None: + self._pipelines.cleanup_runtime_caches() + lease.close() def generate(self, req: IcLoraGenerateRequest) -> IcLoraGenerateResponse: if req.ic_lora_id is not None: @@ -443,7 +452,13 @@ def generate(self, req: IcLoraGenerateRequest) -> IcLoraGenerateResponse: t_total_start = time.perf_counter() logger.info("[ic-lora] Generation started (conditioning=%s)", req.conditioning_type) + lease = None try: + lease = self._generation.acquire_local_metal_lease( + generation_id=generation_id, + workload=f"ic_lora:{req.conditioning_type}", + reason="Torch local IC-LoRA generation", + ) t_load_start = time.perf_counter() with log_heartbeat("ic-lora model load"): ic_state = self._pipelines.load_ic_lora( @@ -630,3 +645,6 @@ def generate(self, req: IcLoraGenerateRequest) -> IcLoraGenerateResponse: raise HTTPError(500, f"Generation error: {exc}") from exc finally: self._text.clear_api_embeddings() + if lease is not None: + self._pipelines.cleanup_runtime_caches() + lease.close() diff --git a/backend/handlers/image_generation_handler.py b/backend/handlers/image_generation_handler.py index 9024be001..9ddb66599 100644 --- a/backend/handlers/image_generation_handler.py +++ b/backend/handlers/image_generation_handler.py @@ -82,7 +82,13 @@ def generate(self, req: GenerateImageRequest) -> GenerateImageResponse: ) generation_id = uuid.uuid4().hex[:8] + lease = None try: + lease = self._generation.acquire_local_metal_lease( + generation_id=generation_id, + workload="image_generation", + reason="Torch local image generation", + ) self._pipelines.load_image_generation_pipeline_to_gpu() self._generation.start_generation(generation_id) output_paths = self.generate_image( @@ -101,6 +107,10 @@ def generate(self, req: GenerateImageRequest) -> GenerateImageResponse: logger.info("Image generation cancelled by user") return GenerateImageCancelledResponse(status="cancelled") raise HTTPError(500, str(e)) from e + finally: + if lease is not None: + self._pipelines.cleanup_runtime_caches() + lease.close() def _edit( self, @@ -131,7 +141,13 @@ def _edit( ) generation_id = uuid.uuid4().hex[:8] + lease = None try: + lease = self._generation.acquire_local_metal_lease( + generation_id=generation_id, + workload="image_edit", + reason="Torch local image editing", + ) self._pipelines.load_image_generation_pipeline_to_gpu() self._generation.start_generation(generation_id) output_paths = self.edit_image( @@ -150,6 +166,10 @@ def _edit( logger.info("Image edit cancelled by user") return GenerateImageCancelledResponse(status="cancelled") raise HTTPError(500, str(e)) from e + finally: + if lease is not None: + self._pipelines.cleanup_runtime_caches() + lease.close() def edit_image( self, diff --git a/backend/handlers/pipelines_handler.py b/backend/handlers/pipelines_handler.py index 1be340ccc..51f0c9ee1 100644 --- a/backend/handlers/pipelines_handler.py +++ b/backend/handlers/pipelines_handler.py @@ -4,7 +4,7 @@ import logging from threading import RLock -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal from _routes._errors import HTTPError from api_types import LTXLocalModelId @@ -16,7 +16,10 @@ get_ltx_model_spec, resolve_active_ltx_model_id, ) -from runtime_config.runtime_policy import streaming_prefetch_count_for_mode +from runtime_config.runtime_policy import ( + decide_fast_video_execution_mode, + streaming_prefetch_count_for_mode, +) from services.interfaces import ( A2VPipeline, DepthProcessorPipeline, @@ -55,6 +58,7 @@ def __init__( text_handler: TextHandler, gpu_cleaner: GpuCleaner, fast_video_pipeline_class: type[FastVideoPipeline], + mlx_fast_video_pipeline_class: type[FastVideoPipeline], image_generation_pipeline_class: type[ImageGenerationPipeline], ic_lora_pipeline_class: type[IcLoraPipeline], depth_processor_pipeline_class: type[DepthProcessorPipeline], @@ -67,6 +71,7 @@ def __init__( self._text_handler = text_handler self._gpu_cleaner = gpu_cleaner self._fast_video_pipeline_class = fast_video_pipeline_class + self._mlx_fast_video_pipeline_class = mlx_fast_video_pipeline_class self._image_generation_pipeline_class = image_generation_pipeline_class self._ic_lora_pipeline_class = ic_lora_pipeline_class self._depth_processor_pipeline_class = depth_processor_pipeline_class @@ -82,10 +87,14 @@ def _ensure_no_running_generation(self) -> None: case _: return - def _pipeline_matches_model_type(self, model_type: VideoPipelineModelType) -> bool: + def _pipeline_matches_model_type( + self, + model_type: VideoPipelineModelType, + runtime_engine: Literal["torch", "mlx"], + ) -> bool: match self.state.gpu_slot: - case GpuSlot(active_pipeline=VideoPipelineState(pipeline=pipeline)): - return pipeline.pipeline_kind == model_type + case GpuSlot(active_pipeline=VideoPipelineState(pipeline=pipeline, runtime_engine=active_engine)): + return pipeline.pipeline_kind == model_type and active_engine == runtime_engine case _: return False @@ -114,6 +123,8 @@ def _require_downloaded_ltx_model_id(self) -> LTXLocalModelId: return model_id def _compile_if_enabled(self, state: VideoPipelineState) -> VideoPipelineState: + if state.runtime_engine == "mlx": + return state if not self.state.app_settings.use_torch_compile: return state if state.is_compiled: @@ -144,7 +155,10 @@ def _compile_if_enabled(self, state: VideoPipelineState) -> VideoPipelineState: return state def _create_video_pipeline( - self, model_type: VideoPipelineModelType, loras: list[tuple[str, float]] | None = None + self, + model_type: VideoPipelineModelType, + loras: list[tuple[str, float]] | None = None, + runtime_engine: Literal["torch", "mlx"] = "torch", ) -> VideoPipelineState: gemma_root = self._text_handler.resolve_gemma_root() model_id = self._require_downloaded_ltx_model_id() @@ -152,18 +166,36 @@ def _create_video_pipeline( checkpoint_path = str(get_existing_cp_path(self.models_dir, spec.model_cp)) upsampler_path = str(get_existing_cp_path(self.models_dir, spec.upscale_cp)) - pipeline = self._fast_video_pipeline_class.create( + pipeline_class = ( + self._mlx_fast_video_pipeline_class + if runtime_engine == "mlx" + else self._fast_video_pipeline_class + ) + if runtime_engine == "mlx": + execution_mode = decide_fast_video_execution_mode( + "mlx", + self.config.local_generations_mode, + self.config.available_ram_gb, + ) + streaming_prefetch_count = 2 if execution_mode == "low_ram" else None + else: + streaming_prefetch_count = streaming_prefetch_count_for_mode( + self.config.local_generations_mode + ) + + pipeline = pipeline_class.create( checkpoint_path, gemma_root, upsampler_path, self.config.device, - streaming_prefetch_count_for_mode(self.config.local_generations_mode), + streaming_prefetch_count, loras=loras or [], ) state = VideoPipelineState( pipeline=pipeline, is_compiled=False, + runtime_engine=runtime_engine, loras=tuple(loras) if loras else (), gemma_root=gemma_root, ) @@ -176,6 +208,10 @@ def unload_gpu_pipeline(self) -> None: self._assert_invariants() self._gpu_cleaner.cleanup() + def cleanup_runtime_caches(self) -> None: + """Release allocator/cache memory while preserving a reusable pipeline.""" + self._gpu_cleaner.cleanup() + def park_image_generation_pipeline_on_cpu(self) -> None: image_generation_pipeline: ImageGenerationPipeline | None = None @@ -270,6 +306,7 @@ def load_gpu_pipeline( self, model_type: VideoPipelineModelType, loras: list[tuple[str, float]] | None = None, + runtime_engine: Literal["torch", "mlx"] = "torch", ) -> VideoPipelineState: self._install_text_patches_if_needed() @@ -277,7 +314,7 @@ def load_gpu_pipeline( requested_gemma_root = self._text_handler.resolve_gemma_root() state: VideoPipelineState | None = None with self._lock: - if self._pipeline_matches_model_type(model_type): + if self._pipeline_matches_model_type(model_type, runtime_engine): match self.state.gpu_slot: case GpuSlot( active_pipeline=VideoPipelineState() as existing_state @@ -291,7 +328,11 @@ def load_gpu_pipeline( if state is None: self._evict_gpu_pipeline_for_swap() - state = self._create_video_pipeline(model_type, loras=loras) + state = self._create_video_pipeline( + model_type, + loras=loras, + runtime_engine=runtime_engine, + ) with self._lock: self.state.gpu_slot = GpuSlot(active_pipeline=state) self._assert_invariants() diff --git a/backend/handlers/retake_handler.py b/backend/handlers/retake_handler.py index d6c5abd13..4c5d6e2d5 100644 --- a/backend/handlers/retake_handler.py +++ b/backend/handlers/retake_handler.py @@ -172,17 +172,22 @@ def _run_local_retake( target_width, target_height = resolve_target_resolution(resolution, source_width, source_height) - try: - self._text.prepare_text_encoding(prompt, enhance_prompt=False) - except RuntimeError as exc: - raise HTTPError(400, str(exc)) from exc - generation_id = uuid.uuid4().hex[:8] seed = self._resolve_seed() output_path = self.config.outputs_dir / f"retake_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{generation_id}.mp4" regenerate_video, regenerate_audio = self._resolve_retake_mode(mode) + lease = None try: + lease = self._generation.acquire_local_metal_lease( + generation_id=generation_id, + workload="retake", + reason="Torch local retake generation", + ) + try: + self._text.prepare_text_encoding(prompt, enhance_prompt=False) + except RuntimeError as exc: + raise HTTPError(400, str(exc)) from exc pipeline_state = self._pipelines.load_retake_pipeline(distilled=True) self._generation.start_generation(generation_id) self._generation.update_progress("loading_model", 5, 0, 1) @@ -225,6 +230,9 @@ def _run_local_retake( raise HTTPError(500, f"Generation error: {exc}") from exc finally: self._text.clear_api_embeddings() + if lease is not None: + self._pipelines.cleanup_runtime_caches() + lease.close() @staticmethod def _resolve_retake_mode(mode: RetakeMode) -> tuple[bool, bool]: diff --git a/backend/handlers/runtime_policy_handler.py b/backend/handlers/runtime_policy_handler.py index e86e56f9e..cab744a54 100644 --- a/backend/handlers/runtime_policy_handler.py +++ b/backend/handlers/runtime_policy_handler.py @@ -2,8 +2,23 @@ from __future__ import annotations -from api_types import RuntimePolicyResponse +from importlib.metadata import PackageNotFoundError, version + +from api_types import RuntimeEngine, RuntimePolicyResponse, RuntimeProvenanceItem from runtime_config.runtime_config import RuntimeConfig +from runtime_config.runtime_policy import ( + MLX_RUNTIME_REVISION, + MLX_RUNTIME_VERSION, + TORCH_RUNTIME_REVISION, + decide_fast_video_execution_mode, +) + + +def _package_version(package: str) -> str: + try: + return version(package) + except PackageNotFoundError: + return "not installed" class RuntimePolicyHandler: @@ -11,5 +26,90 @@ def __init__(self, config: RuntimeConfig) -> None: self._config = config def get_runtime_policy(self) -> RuntimePolicyResponse: - # Server-side single source of truth for forced API mode. - return RuntimePolicyResponse(force_api_generations=self._config.force_api_generations) + auto_decision = self._config.decide_fast_video_engine(use_local_text_encoding=True) + execution_mode = decide_fast_video_execution_mode( + auto_decision.engine, + self._config.local_generations_mode, + self._config.available_ram_gb, + ) + if self._config.force_api_generations: + auto_engine = "cloud" + capability_engines: dict[str, RuntimeEngine] = { + "fast_t2v_i2v": "cloud", + "prepared_text_embeddings": "cloud", + "audio_to_video": "cloud", + "retake": "cloud", + "extend": "cloud", + "ic_lora": "cloud", + "image_generation": "cloud", + } + else: + auto_engine = auto_decision.engine + capability_engines = { + "fast_t2v_i2v": auto_decision.engine, + "prepared_text_embeddings": "torch", + "audio_to_video": "torch", + "retake": "torch", + "extend": "torch", + "ic_lora": "torch", + "image_generation": "torch", + } + + quality_warning = None + if self._config.mlx_model_variant == "q8": + quality_warning = ( + "MLX q8 is an expert-only option and is not auto-selected because " + "quality qualification found measurable video/audio loss." + ) + + return RuntimePolicyResponse( + force_api_generations=self._config.force_api_generations, + fast_video_engine_preference=self._config.fast_video_engine_preference, + auto_fast_video_engine=auto_engine, + auto_selection_reason=auto_decision.reason, + execution_mode=execution_mode, + automatic_tiling=execution_mode == "low_ram", + mlx_model_source=self._config.mlx_model_source, + mlx_model_variant=self._config.mlx_model_variant, + quality_warning=quality_warning, + capability_engines=capability_engines, + provenance=[ + RuntimeProvenanceItem( + component="ltx-pipelines", + version=_package_version("ltx-pipelines"), + revision=TORCH_RUNTIME_REVISION, + source="https://github.com/Lightricks/LTX-2", + ), + RuntimeProvenanceItem( + component="torch", + version=_package_version("torch"), + source="https://pytorch.org", + ), + RuntimeProvenanceItem( + component=( + "ltx-pipelines-mlx sidecar (actual, dirty)" + if self._config.mlx_runtime_dirty + else "ltx-pipelines-mlx sidecar (actual)" + ), + version=self._config.mlx_runtime_version, + revision=self._config.mlx_runtime_revision, + source=self._config.mlx_runtime_source, + ), + RuntimeProvenanceItem( + component="ltx-pipelines-mlx compatibility target", + version=MLX_RUNTIME_VERSION, + revision=MLX_RUNTIME_REVISION, + source="https://github.com/dgrauet/ltx-2-mlx", + ), + RuntimeProvenanceItem( + component="ltx-core-mlx sidecar dependency", + version=self._config.mlx_core_version or "not reported", + source=self._config.mlx_runtime_source, + ), + RuntimeProvenanceItem( + component="mlx sidecar framework", + version=self._config.mlx_framework_version or "not reported", + source=self._config.mlx_runtime_source, + ), + ], + ) diff --git a/backend/handlers/video_generation_handler.py b/backend/handlers/video_generation_handler.py index f0c861341..cc173e05b 100644 --- a/backend/handlers/video_generation_handler.py +++ b/backend/handlers/video_generation_handler.py @@ -13,6 +13,7 @@ from pathlib import Path from threading import RLock from typing import TYPE_CHECKING +from typing import Literal from PIL import Image @@ -37,6 +38,7 @@ from handlers.generation_handler import GenerationHandler from handlers.pipelines_handler import PipelinesHandler from handlers.text_handler import TextHandler +from handlers.video_resolution import resolve_fast_video_dimensions from server_utils.media_validation import ( normalize_optional_path, validate_audio_file, @@ -109,27 +111,10 @@ def generate(self, req: GenerateVideoRequest) -> GenerateVideoResponse: logger.info("Resolution %s - using fast pipeline", resolution) - RESOLUTION_MAP_16_9: dict[str, tuple[int, int]] = { - "540p": (960, 544), - "720p": (1280, 704), - "1080p": (1920, 1088), - } - - def get_16_9_size(res: str) -> tuple[int, int]: - size = RESOLUTION_MAP_16_9.get(res) - if size is None: - raise HTTPError(400, "INVALID_LOCAL_RESOLUTION") - return size - - def get_9_16_size(res: str) -> tuple[int, int]: - w, h = get_16_9_size(res) - return h, w - - match req.aspectRatio: - case "9:16": - width, height = get_9_16_size(resolution) - case "16:9": - width, height = get_16_9_size(resolution) + try: + width, height = resolve_fast_video_dimensions(resolution, req.aspectRatio) + except ValueError as exc: + raise HTTPError(400, "INVALID_LOCAL_RESOLUTION") from exc num_frames = self._compute_num_frames(duration, fps) @@ -142,26 +127,61 @@ def get_9_16_size(res: str) -> tuple[int, int]: generation_id = self._make_generation_id() seed = req.seed if req.seed is not None else self._resolve_seed() loras = self._resolve_loras(req.loras) + use_local_text_encoding = self._text.should_use_local_encoding() + engine_decision = self.config.decide_fast_video_engine( + use_local_text_encoding=use_local_text_encoding, + ) + runtime_engine = engine_decision.engine + logger.info( + "[%s] Fast runtime selected: %s (%s)", + "i2v" if image is not None else "t2v", + runtime_engine, + engine_decision.reason, + ) try: - self._pipelines.load_gpu_pipeline("fast", loras=loras) - self._generation.start_generation(generation_id) - - output_path = self.generate_video( - prompt=req.prompt, - image=image, - height=height, - width=width, - num_frames=num_frames, - fps=fps, - seed=seed, - camera_motion=req.cameraMotion, - negative_prompt=req.negativePrompt, - loras=loras, - ) + with self._generation.hold_local_metal_lease( + generation_id=generation_id, + workload=f"fast_{'i2v' if image is not None else 't2v'}", + reason=f"{runtime_engine} local video generation", + ): + try: + self._pipelines.load_gpu_pipeline( + "fast", + loras=loras, + runtime_engine=runtime_engine, + ) + self._generation.start_generation(generation_id) + + output_path = self.generate_video( + prompt=req.prompt, + image=image, + height=height, + width=width, + num_frames=num_frames, + fps=fps, + seed=seed, + camera_motion=req.cameraMotion, + negative_prompt=req.negativePrompt, + loras=loras, + runtime_engine=runtime_engine, + ) - self._generation.complete_generation(output_path) - return GenerateVideoCompleteResponse(status="complete", video_path=output_path) + self._generation.complete_generation(output_path) + return GenerateVideoCompleteResponse( + status="complete", + video_path=output_path, + resolved_width=width, + resolved_height=height, + ) + except Exception as exc: + # Transition out of GenerationRunning before teardown so + # unload_gpu_pipeline can release Metal memory while the + # shared lease is still held. + self._generation.fail_generation(str(exc)) + raise + finally: + self._pipelines.cleanup_runtime_caches() except HTTPError as e: self._generation.fail_generation(e.detail) @@ -192,6 +212,7 @@ def generate_video( camera_motion: VideoCameraMotion, negative_prompt: str, loras: list[tuple[str, float]] | None = None, + runtime_engine: Literal["torch", "mlx"] = "torch", ) -> str: t_total_start = time.perf_counter() gen_mode = "i2v" if image is not None else "t2v" @@ -204,7 +225,11 @@ def generate_video( self._generation.update_progress("loading_model", 5, 0, total_steps) t_load_start = time.perf_counter() - pipeline_state = self._pipelines.load_gpu_pipeline("fast", loras=loras) + pipeline_state = self._pipelines.load_gpu_pipeline( + "fast", + loras=loras, + runtime_engine=runtime_engine, + ) t_load_end = time.perf_counter() logger.info("[%s] Pipeline load: %.2fs", gen_mode, t_load_end - t_load_start) @@ -223,22 +248,28 @@ def generate_video( try: settings = self.state.app_settings - use_api_encoding = not self._text.should_use_local_encoding() - if image is not None: - enhance = use_api_encoding and settings.prompt_enhancer_enabled_i2v - else: - enhance = use_api_encoding and settings.prompt_enhancer_enabled_t2v - - encoding_method = "api" if use_api_encoding else "local" t_text_start = time.perf_counter() - self._text.prepare_text_encoding(enhanced_prompt, enhance_prompt=enhance) + if runtime_engine == "mlx": + # MLX owns its 4-bit Gemma encoding lifecycle and cannot consume + # the Torch pipeline's prepared/API embedding tensors. + encoding_method = "mlx-local" + else: + use_api_encoding = not self._text.should_use_local_encoding() + if image is not None: + enhance = use_api_encoding and settings.prompt_enhancer_enabled_i2v + else: + enhance = use_api_encoding and settings.prompt_enhancer_enabled_t2v + encoding_method = "api" if use_api_encoding else "local" + self._text.prepare_text_encoding(enhanced_prompt, enhance_prompt=enhance) t_text_end = time.perf_counter() logger.info("[%s] Text encoding (%s): %.2fs", gen_mode, encoding_method, t_text_end - t_text_start) self._generation.update_progress("inference", 15, 0, total_steps) - height = round(height / 64) * 64 - width = round(width / 64) * 64 + if height % 64 or width % 64: + raise RuntimeError( + f"Fast two-stage dimensions must use the 64-pixel grid; got {width}x{height}" + ) t_inference_start = time.perf_counter() with log_heartbeat(f"{gen_mode} inference"): @@ -303,7 +334,13 @@ def _generate_a2v( generation_id = self._make_generation_id() + lease = None try: + lease = self._generation.acquire_local_metal_lease( + generation_id=generation_id, + workload="audio_to_video", + reason="Torch local audio-to-video generation", + ) a2v_state = self._pipelines.load_a2v_pipeline(loras=loras) self._generation.start_generation(generation_id) @@ -370,6 +407,9 @@ def _generate_a2v( self._text.clear_api_embeddings() if temp_image_path and os.path.exists(temp_image_path): os.unlink(temp_image_path) + if lease is not None: + self._pipelines.cleanup_runtime_caches() + lease.close() def _prepare_image(self, image_path: str, width: int, height: int) -> Image.Image: validated_path = validate_image_file(image_path) diff --git a/backend/handlers/video_resolution.py b/backend/handlers/video_resolution.py index 3a5bf24e2..95f5f2278 100644 --- a/backend/handlers/video_resolution.py +++ b/backend/handlers/video_resolution.py @@ -1,9 +1,9 @@ -"""Local source-video prep shared by the retake and extend local paths. +"""Authoritative local video resolution and frame-grid policies. -The local pipeline needs width/height divisible by 32 and a frame count of the form -``8k+1``. Rather than rejecting inputs that don't comply (standard 1080p isn't ÷32 in -height; arbitrary clips rarely land on 8k+1), we correct both: snap resolution down to a -÷32 size (never above source) and trim the frame count down to the nearest ``8k+1``. +Retake and extend never upscale source media and use the VAE's 32-pixel grid. Fast +T2V/I2V is two-stage, so both final axes must already be on its 64-pixel grid; those +catalog sizes are resolved before image preparation or inference and are never silently +rounded at the pipeline boundary. """ from __future__ import annotations @@ -17,6 +17,18 @@ from api_types import TargetResolution _SPATIAL_FACTOR = 32 +_FAST_TWO_STAGE_GRID = 2 * _SPATIAL_FACTOR + +# 540p cannot be represented near 960x540 with two independently 64-aligned axes. +# 896x512 stays under the nominal 960x544 pixel budget and has 1.56% aspect error, +# versus 5.47% for the old silently-rounded 960x512. 1024x576 is exact 16:9 but +# costs 12.9% more pixels than the nominal request. Higher tiers are already valid. +_FAST_LANDSCAPE_DIMENSIONS: dict[str, tuple[int, int]] = { + "540p": (896, 512), + "720p": (1280, 704), + "1080p": (1920, 1088), +} + # VAE temporal downscale: valid frame counts are 8k+1. Single source of truth for the # 8-frame rule (extend's snap-up and video generation's frame math reuse this). TIME_FACTOR = 8 @@ -29,6 +41,25 @@ _ALLOWED_SOURCE_SUFFIXES = frozenset({".mp4", ".mov", ".avi", ".webm", ".mkv"}) +def resolve_fast_video_dimensions(resolution: str, aspect_ratio: str) -> tuple[int, int]: + """Return the pre-qualified two-stage output size as ``(width, height)``. + + The mapping is aspect-aware and budget-conscious rather than independently + rounding each edge, which can silently distort the requested framing. + """ + try: + width, height = _FAST_LANDSCAPE_DIMENSIONS[resolution] + except KeyError as exc: + raise ValueError(f"Unsupported local Fast resolution: {resolution}") from exc + if aspect_ratio == "9:16": + width, height = height, width + elif aspect_ratio != "16:9": + raise ValueError(f"Unsupported local Fast aspect ratio: {aspect_ratio}") + if width % _FAST_TWO_STAGE_GRID or height % _FAST_TWO_STAGE_GRID: + raise AssertionError("Fast video dimension catalog must stay on the 64-pixel grid") + return width, height + + def validate_source_video_path(video_path: str) -> Path: """Resolve and sanity-check a caller-supplied source video path before use.""" if not video_path: diff --git a/backend/ltx2_server.py b/backend/ltx2_server.py index 4878ec412..c7e763470 100644 --- a/backend/ltx2_server.py +++ b/backend/ltx2_server.py @@ -200,7 +200,14 @@ def _resolve_app_data_dir() -> Path: from app_factory import DEFAULT_ALLOWED_ORIGINS, create_app from state import RuntimeConfig, build_initial_state -from runtime_config.runtime_policy import LocalGenerationMode, decide_local_generation_mode +from runtime_config.mlx_runtime import discover_mlx_runtime +from runtime_config.runtime_policy import ( + FastVideoEnginePreference, + LocalGenerationMode, + MLX_BF16_MODEL_SOURCE, + MLX_Q8_MODEL_SOURCE, + decide_local_generation_mode, +) from server_utils.model_layout_migration import migrate_legacy_models_layout from services.gpu_info.gpu_info_impl import GpuInfoImpl @@ -243,6 +250,64 @@ def _resolve_local_generations_mode() -> LocalGenerationMode: LOCAL_GENERATIONS_MODE = _resolve_local_generations_mode() + +def _resolve_fast_video_engine_preference() -> FastVideoEnginePreference: + raw = os.environ.get("LTX_FAST_VIDEO_ENGINE", "auto").strip().lower() + if raw in {"auto", "torch", "mlx"}: + return cast(FastVideoEnginePreference, raw) + logger.warning("Ignoring invalid LTX_FAST_VIDEO_ENGINE=%r; using auto", raw) + return "auto" + + +def _is_mlx_model_cached(model_source: str) -> bool: + local_path = Path(model_source).expanduser() + if local_path.exists(): + return True + try: + from huggingface_hub import scan_cache_dir + + return any(repo.repo_id == model_source for repo in scan_cache_dir().repos) + except Exception: + logger.warning("Failed to inspect the Hugging Face cache for %s", model_source, exc_info=True) + return False + + +FAST_VIDEO_ENGINE_PREFERENCE = _resolve_fast_video_engine_preference() +_mlx_model_variant_raw = os.environ.get("LTX_MLX_MODEL_VARIANT", "bf16").strip().lower() +if _mlx_model_variant_raw not in {"bf16", "q8"}: + logger.warning("Ignoring invalid LTX_MLX_MODEL_VARIANT=%r; using bf16", _mlx_model_variant_raw) + _mlx_model_variant_raw = "bf16" +MLX_MODEL_VARIANT = _mlx_model_variant_raw +MLX_MODEL_SOURCE = os.environ.get( + "LTX_MLX_MODEL_ID", + MLX_Q8_MODEL_SOURCE if MLX_MODEL_VARIANT == "q8" else MLX_BF16_MODEL_SOURCE, +) +_gpu_info_for_runtime = GpuInfoImpl() +AVAILABLE_RAM_GB = ( + _gpu_info_for_runtime.get_available_ram_gb() + if platform.system() == "Darwin" + else None +) +MLX_RUNTIME = discover_mlx_runtime() +MLX_RUNTIME_ELIGIBLE = bool( + platform.system() == "Darwin" + and platform.machine().lower() in {"arm64", "aarch64"} + and _gpu_info_for_runtime.get_mps_available() + and MLX_RUNTIME.compatible +) +MLX_MODEL_CACHED = _is_mlx_model_cached(MLX_MODEL_SOURCE) +logger.info( + "Fast runtime preference=%s mlx_eligible=%s runtime_version=%s runtime_revision=%s " + "model=%s cached=%s variant=%s", + FAST_VIDEO_ENGINE_PREFERENCE, + MLX_RUNTIME_ELIGIBLE, + MLX_RUNTIME.version, + MLX_RUNTIME.revision, + MLX_MODEL_SOURCE, + MLX_MODEL_CACHED, + MLX_MODEL_VARIANT, +) + CAMERA_MOTION_PROMPTS = { "none": "", "static": ", static camera, locked off shot, no camera movement", @@ -273,6 +338,18 @@ def _resolve_local_generations_mode() -> LocalGenerationMode: dev_mode=os.environ.get("LTX_DEV_MODE") == "1", hf_oauth_client_id=HF_OAUTH_CLIENT_ID, backend_port=int(os.environ.get("LTX_PORT", "") or PORT), + fast_video_engine_preference=FAST_VIDEO_ENGINE_PREFERENCE, + mlx_runtime_eligible=MLX_RUNTIME_ELIGIBLE, + mlx_model_cached=MLX_MODEL_CACHED, + mlx_model_source=MLX_MODEL_SOURCE, + mlx_model_variant=cast(Any, MLX_MODEL_VARIANT), + mlx_runtime_version=MLX_RUNTIME.version, + mlx_runtime_revision=MLX_RUNTIME.revision, + mlx_runtime_source=MLX_RUNTIME.source, + mlx_runtime_dirty=MLX_RUNTIME.dirty, + mlx_core_version=MLX_RUNTIME.core_version, + mlx_framework_version=MLX_RUNTIME.mlx_version, + available_ram_gb=AVAILABLE_RAM_GB, lora_catalog_source=str(Path(__file__).parent / "runtime_config" / "lora_catalog.json"), lora_catalog_fallback_path=str(Path(__file__).parent / "runtime_config" / "lora_catalog.json"), ) diff --git a/backend/performance_runner/__init__.py b/backend/performance_runner/__init__.py new file mode 100644 index 000000000..fd484dafb --- /dev/null +++ b/backend/performance_runner/__init__.py @@ -0,0 +1 @@ +"""App-entrypoint performance and qualification tools.""" diff --git a/backend/performance_runner/analyze_metal_trace.py b/backend/performance_runner/analyze_metal_trace.py new file mode 100644 index 000000000..a59bb1fe9 --- /dev/null +++ b/backend/performance_runner/analyze_metal_trace.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Summarize one target process from an xctrace Metal GPU-interval export.""" +from __future__ import annotations + +import argparse +import json +import math +import re +import statistics +import xml.etree.ElementTree as ET +from collections import Counter, defaultdict +from pathlib import Path +from typing import Any + + +def _cell_value(cell: ET.Element, definitions: dict[str, dict[str, Any]]) -> dict[str, Any]: + ref = cell.get("ref") + if ref is not None: + return definitions.get(ref, {"tag": cell.tag, "raw": None, "fmt": None}) + value = { + "tag": cell.tag, + "raw": (cell.text or "").strip() or None, + "fmt": cell.get("fmt"), + } + identifier = cell.get("id") + if identifier is not None: + definitions[identifier] = value + return value + + +def _number(value: dict[str, Any]) -> int | None: + raw = value.get("raw") + try: + return int(raw) if raw is not None else None + except (TypeError, ValueError): + return None + + +def _label_group(label: str) -> str: + value = re.sub(r"\s+\(\s*python[^)]*\([^)]*\)\s*\)\s+0x[0-9a-f]+$", "", label, flags=re.I) + value = re.sub(r"\s+0x[0-9a-f]+$", "", value, flags=re.I) + return value.strip() or "(unlabeled)" + + +def _percentile(values: list[int], percentile: float) -> float | None: + if not values: + return None + ordered = sorted(values) + position = (len(ordered) - 1) * percentile + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return float(ordered[lower]) + return ordered[lower] * (upper - position) + ordered[upper] * (position - lower) + + +def analyze_gpu_intervals(path: Path, target_pid: int) -> dict[str, Any]: + definitions: dict[str, dict[str, Any]] = {} + rows: list[dict[str, Any]] = [] + for _event, element in ET.iterparse(path, events=("end",)): + if element.tag != "row": + continue + for descendant in element.iter(): + identifier = descendant.get("id") + if identifier is not None: + definitions[identifier] = { + "tag": descendant.tag, + "raw": (descendant.text or "").strip() or None, + "fmt": descendant.get("fmt"), + } + cells = [_cell_value(cell, definitions) for cell in element] + if len(cells) < 18: + element.clear() + continue + process = str(cells[10].get("fmt") or "") + if not process.endswith(f"({target_pid})"): + element.clear() + continue + start_ns = _number(cells[0]) + duration_ns = _number(cells[1]) + if start_ns is None or duration_ns is None: + element.clear() + continue + rows.append( + { + "start_ns": start_ns, + "duration_ns": duration_ns, + "channel": cells[2].get("fmt") or cells[2].get("raw") or "unknown", + "depth": _number(cells[5]), + "label": cells[6].get("fmt") or cells[6].get("raw") or "", + "command_buffer_id": cells[15].get("fmt") or cells[15].get("raw"), + "encoder_id": cells[16].get("fmt") or cells[16].get("raw"), + "submission_id": cells[17].get("fmt") or cells[17].get("raw"), + } + ) + element.clear() + + if not rows: + raise ValueError(f"no Metal GPU intervals found for PID {target_pid}") + + durations = [row["duration_ns"] for row in rows] + channel_counts: Counter[str] = Counter() + channel_duration: defaultdict[str, int] = defaultdict(int) + label_counts: Counter[str] = Counter() + label_duration: defaultdict[str, int] = defaultdict(int) + for row in rows: + channel_counts[row["channel"]] += 1 + channel_duration[row["channel"]] += row["duration_ns"] + group = _label_group(row["label"]) + label_counts[group] += 1 + label_duration[group] += row["duration_ns"] + + top_dispatches = sorted(rows, key=lambda row: row["duration_ns"], reverse=True)[:25] + start_ns = min(row["start_ns"] for row in rows) + end_ns = max(row["start_ns"] + row["duration_ns"] for row in rows) + return { + "schema": "ltx.metal-hotspot-report.v1", + "source_export": str(path.resolve()), + "target_pid": target_pid, + "interval_count": len(rows), + "trace_window_seconds": (end_ns - start_ns) / 1e9, + "sum_interval_seconds": sum(durations) / 1e9, + "duration_ms": { + "mean": statistics.fmean(durations) / 1e6, + "median": statistics.median(durations) / 1e6, + "p95": (_percentile(durations, 0.95) or 0) / 1e6, + "p99": (_percentile(durations, 0.99) or 0) / 1e6, + "max": max(durations) / 1e6, + }, + "channels": [ + { + "channel": channel, + "count": channel_counts[channel], + "sum_interval_seconds": channel_duration[channel] / 1e9, + } + for channel in sorted(channel_counts, key=channel_duration.get, reverse=True) + ], + "label_groups": [ + { + "label": label, + "count": label_counts[label], + "sum_interval_seconds": label_duration[label] / 1e9, + } + for label in sorted(label_counts, key=label_duration.get, reverse=True)[:20] + ], + "top_dispatches": [ + { + **row, + "start_seconds": row["start_ns"] / 1e9, + "duration_ms": row["duration_ns"] / 1e6, + } + for row in top_dispatches + ], + "limitations": [ + "Metal System Trace recorded Shader Timeline: Disabled, so kernel symbols and per-shader counters are unavailable.", + "GPU intervals can be nested; summed interval duration is not GPU wall time or utilization.", + "Rows are filtered to the target child PID so unrelated system Metal activity is excluded.", + ], + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("export", type=Path) + parser.add_argument("--pid", type=int, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + report = analyze_gpu_intervals(args.export, args.pid) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print(args.output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_dimensions/fast_t2v_540p_5s/request.json b/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_dimensions/fast_t2v_540p_5s/request.json new file mode 100644 index 000000000..fd280911d --- /dev/null +++ b/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_dimensions/fast_t2v_540p_5s/request.json @@ -0,0 +1,15 @@ +{ + "prompt": "A slow, steady lateral camera move reveals the scene with natural subject motion and physically consistent lighting.", + "resolution": "540p", + "model": "fast", + "cameraMotion": "none", + "negativePrompt": "", + "duration": 5, + "fps": 24, + "audio": false, + "imagePath": null, + "audioPath": null, + "aspectRatio": "16:9", + "seed": 424242, + "loras": [] +} \ No newline at end of file diff --git a/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_dimensions/fast_t2v_540p_5s/result.json b/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_dimensions/fast_t2v_540p_5s/result.json new file mode 100644 index 000000000..22109cde3 --- /dev/null +++ b/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_dimensions/fast_t2v_540p_5s/result.json @@ -0,0 +1,254 @@ +{ + "schema": "ltx.hd-benchmark.result.v1", + "product": "ltx_desktop_electron", + "case": { + "case_id": "fast_t2v_540p_5s", + "resolution": "540p", + "duration": 5, + "fps": 24, + "aspect_ratio": "16:9", + "image_conditioned": false + }, + "response": { + "status": "complete", + "video_path": "/Users//Library/Application Support/LTXDesktop/outputs/ltx2_video_20260726_190747_5f235736.mp4", + "resolved_width": 896, + "resolved_height": 512 + }, + "submitted_at": 1785114467.541415, + "finished_at": 1785114534.149004, + "wall_seconds": 66.60758900642395, + "output_path": "/Users//Library/Application Support/LTXDesktop/outputs/ltx2_video_20260726_190747_5f235736.mp4", + "runtime_policy": { + "force_api_generations": false, + "fast_video_engine_preference": "auto", + "auto_fast_video_engine": "mlx", + "auto_selection_reason": "MLX auto-selected for Fast T2V/I2V with local text encoding and cached BF16 weights.", + "execution_mode": "eager", + "automatic_tiling": false, + "mlx_model_source": "dgrauet/ltx-2.3-mlx", + "mlx_model_variant": "bf16", + "quality_warning": null, + "capability_engines": { + "fast_t2v_i2v": "mlx", + "prepared_text_embeddings": "torch", + "audio_to_video": "torch", + "retake": "torch", + "extend": "torch", + "ic_lora": "torch", + "image_generation": "torch" + }, + "provenance": [ + { + "component": "ltx-pipelines", + "version": "1.1.7", + "revision": "9377758131b1ffde4b7f766804590a6617bf2ab9", + "source": "https://github.com/Lightricks/LTX-2" + }, + { + "component": "torch", + "version": "2.11.0", + "revision": null, + "source": "https://pytorch.org" + }, + { + "component": "ltx-pipelines-mlx sidecar (actual)", + "version": "0.14.20.dev1", + "revision": "3171bac4ba901c0237faea2678c34034b37abc2a", + "source": "LTX_MLX_PYTHON" + }, + { + "component": "ltx-pipelines-mlx compatibility target", + "version": "0.14.20.dev1", + "revision": "3171bac4ba901c0237faea2678c34034b37abc2a", + "source": "https://github.com/dgrauet/ltx-2-mlx" + }, + { + "component": "ltx-core-mlx sidecar dependency", + "version": "0.14.20.dev1", + "revision": null, + "source": "LTX_MLX_PYTHON" + }, + { + "component": "mlx sidecar framework", + "version": "0.31.1", + "revision": null, + "source": "LTX_MLX_PYTHON" + } + ] + }, + "progress_phases": [ + "inference" + ], + "cleanup_evidence": [ + { + "timestamp": 1785114467.5459058, + "sampled_at": "2026-07-27T01:07:47.545264+00:00", + "active_engine": null, + "active_pipeline": null, + "process_rss_mib": 498, + "system_total_mib": 131072, + "system_available_mib": 84901, + "mlx_active_mib": null, + "mlx_cache_mib": null, + "mlx_peak_mib": null, + "mlx_profile_status": null, + "mlx_profile_phase": null, + "mlx_profile_path": null, + "mlx_profile_sampled_at": null, + "mlx_runtime_identity": null, + "mps_allocated_mib": 0, + "mps_driver_mib": 8, + "mps_recommended_max_mib": 110100, + "local_metal_lease_status": "held", + "local_metal_lease_reason": "mlx local video generation", + "local_metal_lease_waited_seconds": 2.4250010028481483e-05, + "local_metal_lease_owner": { + "schema": "ltx.local-metal-lock.v1", + "product": "LTX Desktop", + "pid": 61831, + "ppid": 61825, + "job_id": "e787a3f3", + "workload": "fast_t2v", + "reason": "mlx local video generation", + "acquired_at_utc": "2026-07-27T01:07:47.543803+00:00", + "host": ".local" + } + } + ], + "dimensions": { + "requested": { + "resolution": "540p", + "duration": 5, + "fps": 24, + "aspect_ratio": "16:9" + }, + "resolved": { + "width": 896, + "height": 512, + "frames": 121, + "execution_mode": "eager" + }, + "actual": { + "width": 896, + "height": 512, + "frames": 121, + "duration_seconds": 5.041667, + "has_audio": true, + "video_codec": "h264", + "audio_codec": "aac" + } + }, + "hashes": { + "recipe_sha256": "5e70e257d4ec400a9d633311b2d0bec2661422e24bdb135ec3a6c5b4e2e522a3", + "prompt_sha256": "9484ef8602be5a89d2d1e6920d3f8bb155458f4c052e1dbedc3971c6628801ee", + "source_sha256": "bc410ed908e0cd7bb2ecf791eb66f9e8d6a83c401bf85e50485e510667fedf77", + "output_sha256": "5f1f02369e48d61b2951bc4d5b38321d4e6d22c65dee2e4870d6ee20de15185c", + "repo_head": "f539eb6277370c2d1714590b31c76ab1f4b95f6b", + "runtime_policy_sha256": "12802c412c09a825e4ff5698e8a3b7bda692790c8c16e512c6c52d90d47782cc" + }, + "lease": { + "initial_probe": { + "schema": "ltx.benchmark.lock-probe.v1", + "lease_path": "/Users//Library/Application Support/LTX Shared/local-metal.lock", + "observed": "acquired", + "holder_payload": { + "acquired_at_utc": "2026-07-27T01:02:26.129668+00:00", + "host": ".local", + "job_id": "9a498577", + "pid": 60053, + "ppid": 60035, + "product": "LTX Desktop", + "reason": "Torch local image generation", + "schema": "ltx.local-metal-lock.v1", + "workload": "image_generation" + }, + "holder_payload_schema_valid": true, + "timestamp": 1785114467.541412, + "monotonic_ns": 346573497311166 + }, + "running_probe": { + "schema": "ltx.benchmark.lock-probe.v1", + "lease_path": "/Users//Library/Application Support/LTX Shared/local-metal.lock", + "observed": "contended", + "holder_payload": { + "acquired_at_utc": "2026-07-27T01:07:47.543803+00:00", + "host": ".local", + "job_id": "e787a3f3", + "pid": 61831, + "ppid": 61825, + "product": "LTX Desktop", + "reason": "mlx local video generation", + "schema": "ltx.local-metal-lock.v1", + "workload": "fast_t2v" + }, + "holder_payload_schema_valid": true, + "timestamp": 1785114467.5459661, + "monotonic_ns": 346573501865250 + }, + "terminal_probe": { + "schema": "ltx.benchmark.lock-probe.v1", + "lease_path": "/Users//Library/Application Support/LTX Shared/local-metal.lock", + "observed": "acquired", + "holder_payload": { + "acquired_at_utc": "2026-07-27T01:07:47.543803+00:00", + "host": ".local", + "job_id": "e787a3f3", + "pid": 61831, + "ppid": 61825, + "product": "LTX Desktop", + "reason": "mlx local video generation", + "schema": "ltx.local-metal-lock.v1", + "workload": "fast_t2v" + }, + "holder_payload_schema_valid": true, + "timestamp": 1785114533.863661, + "monotonic_ns": 346639819525416 + } + }, + "runtime_summary": { + "samples": 64, + "peak_rss_gib": 0.4873046875, + "peak_mlx_mib": 38585, + "peak_mps_allocated_mib": 0, + "peak_mps_driver_mib": 8, + "peak_process_tree_rss_gib": 34.135528564453125, + "peak_physical_footprint_gib": 41.133451238274574, + "peak_cpu_percent": 290.7, + "peak_gpu_utilization_percent": 100.0, + "mlx_profile_status": "success", + "mlx_profile_phase": "Decoding video + audio + muxing", + "mlx_profile_path": "/Users//Library/Application Support/LTXDesktop/outputs/.mlx-profiles/ltx2_video_20260726_190747_5f235736-cb193fdca4204f75bbc898782b7db313.jsonl", + "mlx_profile_sampled_at": "2026-07-27T01:08:53.271645+00:00", + "mlx_runtime_identity": { + "runtime_commit": "3171bac4ba901c0237faea2678c34034b37abc2a", + "runtime_dirty": false, + "runtime_version": "0.14.20.dev1", + "core_version": "0.14.20.dev1", + "mlx_version": "0.31.1", + "mlx_metal_version": "0.31.1", + "device_name": "Apple M5 Max", + "device_architecture": "applegpu_g17s", + "device_memory_bytes": 137438953472, + "device_recommended_working_set_bytes": 115448725504, + "runtime_family": "0.14", + "device_family": "applegpu_g17s" + } + }, + "machine": { + "captured_at_utc": "2026-07-27T01:08:53.901605+00:00", + "host": ".local", + "python": "3.13.12 (main, Mar 25 2026, 02:48:22) [Clang 22.1.1 ]", + "sw_vers": "ProductName:\t\tmacOS\nProductVersion:\t\t26.5.2\nBuildVersion:\t\t25F84", + "hardware": "Hardware:\n\n Hardware Overview:\n\n Model Name: MacBook Pro\n Model Identifier: Mac17,6\n Model Number: Z1N20001GLL/A\n Chip: Apple M5 Max\n Total Number of Cores: 18 (6 Super and 12 Performance)\n Memory: 128 GB\n System Firmware Version: 18000.121.3\n OS Loader Version: 18000.121.3", + "memsize": "137438953472", + "repo_head": "f539eb6277370c2d1714590b31c76ab1f4b95f6b", + "repo_status": "?? backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_dimensions/", + "ffmpeg": [ + "ffmpeg version 8.1.1 Copyright (c) 2000-2026 the FFmpeg developers" + ] + }, + "cancel_sent": false, + "strict_failures": [], + "strict_status": "PASS" +} \ No newline at end of file diff --git a/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_dimensions/plan.json b/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_dimensions/plan.json new file mode 100644 index 000000000..d28ef10a6 --- /dev/null +++ b/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_dimensions/plan.json @@ -0,0 +1,24 @@ +{ + "schema": "ltx.hd-benchmark.plan.v1", + "product": "ltx_desktop_electron", + "production_entrypoint": "POST /api/generate", + "production_lease": "/Users//Library/Application Support/LTX Shared/local-metal.lock", + "expect_mode": "eager", + "runs_per_case": 1, + "cases": [ + { + "case_id": "fast_t2v_540p_5s", + "resolution": "540p", + "duration": 5, + "fps": 24, + "aspect_ratio": "16:9", + "image_conditioned": false + } + ], + "notes": [ + "Run once with --expect-mode=eager and once with --expect-mode=low_ram; runtime policy, not a request-only toggle, owns the app mode.", + "Distilled Fast T2V/I2V cannot use TeaCache and this harness never presents it as a switch.", + "Explicit modality-tiling candidates are qualified in the a second local application MLX HQ matrix; Electron Fast exposes only its production automatic policy.", + "The production flock serializes local heavy work; cloud/API/CPU-only work remains outside it." + ] +} \ No newline at end of file diff --git a/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_dimensions/summary.md b/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_dimensions/summary.md new file mode 100644 index 000000000..2a3f6a7ab --- /dev/null +++ b/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_dimensions/summary.md @@ -0,0 +1,7 @@ +# LTX Desktop Electron HD benchmark + +Command: `/Users//projects/ltx-desktop-electron/backend/performance_runner/hd_matrix.py --execute --expect-mode=eager --case fast_t2v_540p_5s --artifacts performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_dimensions` + +| case | mode | status | wall s | footprint GiB | MLX MiB | SHA-256 | strict gaps | +|---|---|---:|---:|---:|---:|---|---| +| fast_t2v_540p_5s | eager | PASS | 66.61 | 41.133451238274574 | 38585 | 5f1f02369e48d61b2951bc4d5b38321d4e6d22c65dee2e4870d6ee20de15185c | — | diff --git a/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_profiled/fast_t2v_540p_5s/request.json b/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_profiled/fast_t2v_540p_5s/request.json new file mode 100644 index 000000000..fd280911d --- /dev/null +++ b/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_profiled/fast_t2v_540p_5s/request.json @@ -0,0 +1,15 @@ +{ + "prompt": "A slow, steady lateral camera move reveals the scene with natural subject motion and physically consistent lighting.", + "resolution": "540p", + "model": "fast", + "cameraMotion": "none", + "negativePrompt": "", + "duration": 5, + "fps": 24, + "audio": false, + "imagePath": null, + "audioPath": null, + "aspectRatio": "16:9", + "seed": 424242, + "loras": [] +} \ No newline at end of file diff --git a/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_profiled/fast_t2v_540p_5s/result.json b/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_profiled/fast_t2v_540p_5s/result.json new file mode 100644 index 000000000..ee8b7a48a --- /dev/null +++ b/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_profiled/fast_t2v_540p_5s/result.json @@ -0,0 +1,252 @@ +{ + "schema": "ltx.hd-benchmark.result.v1", + "product": "ltx_desktop_electron", + "case": { + "case_id": "fast_t2v_540p_5s", + "resolution": "540p", + "duration": 5, + "fps": 24, + "aspect_ratio": "16:9", + "image_conditioned": false + }, + "response": { + "status": "complete", + "video_path": "/Users//Library/Application Support/LTXDesktop/outputs/ltx2_video_20260726_184911_64fafc87.mp4" + }, + "submitted_at": 1785113351.733394, + "finished_at": 1785113427.746161, + "wall_seconds": 76.01276803016663, + "output_path": "/Users//Library/Application Support/LTXDesktop/outputs/ltx2_video_20260726_184911_64fafc87.mp4", + "runtime_policy": { + "force_api_generations": false, + "fast_video_engine_preference": "auto", + "auto_fast_video_engine": "mlx", + "auto_selection_reason": "MLX auto-selected for Fast T2V/I2V with local text encoding and cached BF16 weights.", + "execution_mode": "eager", + "automatic_tiling": false, + "mlx_model_source": "dgrauet/ltx-2.3-mlx", + "mlx_model_variant": "bf16", + "quality_warning": null, + "capability_engines": { + "fast_t2v_i2v": "mlx", + "prepared_text_embeddings": "torch", + "audio_to_video": "torch", + "retake": "torch", + "extend": "torch", + "ic_lora": "torch", + "image_generation": "torch" + }, + "provenance": [ + { + "component": "ltx-pipelines", + "version": "1.1.7", + "revision": "9377758131b1ffde4b7f766804590a6617bf2ab9", + "source": "https://github.com/Lightricks/LTX-2" + }, + { + "component": "torch", + "version": "2.11.0", + "revision": null, + "source": "https://pytorch.org" + }, + { + "component": "ltx-pipelines-mlx sidecar (actual)", + "version": "0.14.20.dev1", + "revision": "3171bac4ba901c0237faea2678c34034b37abc2a", + "source": "/Users//video-models/ltx-2-mlx/.venv/bin/python" + }, + { + "component": "ltx-pipelines-mlx compatibility target", + "version": "0.14.20.dev1", + "revision": "3171bac4ba901c0237faea2678c34034b37abc2a", + "source": "https://github.com/dgrauet/ltx-2-mlx" + }, + { + "component": "ltx-core-mlx sidecar dependency", + "version": "0.14.20.dev1", + "revision": null, + "source": "/Users//video-models/ltx-2-mlx/.venv/bin/python" + }, + { + "component": "mlx sidecar framework", + "version": "0.31.1", + "revision": null, + "source": "/Users//video-models/ltx-2-mlx/.venv/bin/python" + } + ] + }, + "progress_phases": [ + "inference" + ], + "cleanup_evidence": [ + { + "timestamp": 1785113351.73738, + "sampled_at": "2026-07-27T00:49:11.736730+00:00", + "active_engine": null, + "active_pipeline": null, + "process_rss_mib": 497, + "system_total_mib": 131072, + "system_available_mib": 87487, + "mlx_active_mib": null, + "mlx_cache_mib": null, + "mlx_peak_mib": null, + "mlx_profile_status": null, + "mlx_profile_phase": null, + "mlx_profile_path": null, + "mlx_profile_sampled_at": null, + "mlx_runtime_identity": null, + "mps_allocated_mib": 0, + "mps_driver_mib": 8, + "mps_recommended_max_mib": 110100, + "local_metal_lease_status": "held", + "local_metal_lease_reason": "mlx local video generation", + "local_metal_lease_waited_seconds": 0.00015083304606378078, + "local_metal_lease_owner": { + "schema": "ltx.local-metal-lock.v1", + "product": "LTX Desktop", + "pid": 57580, + "ppid": 57574, + "job_id": "25f4bcac", + "workload": "fast_t2v", + "reason": "mlx local video generation", + "acquired_at_utc": "2026-07-27T00:49:11.735767+00:00", + "host": ".local" + } + } + ], + "dimensions": { + "requested": { + "resolution": "540p", + "duration": 5, + "fps": 24, + "aspect_ratio": "16:9" + }, + "resolved": { + "width": 960, + "height": 544, + "frames": 121, + "execution_mode": "eager" + }, + "actual": { + "width": 960, + "height": 512, + "frames": 121, + "duration_seconds": 5.041667, + "has_audio": true, + "video_codec": "h264", + "audio_codec": "aac" + } + }, + "hashes": { + "recipe_sha256": "5e70e257d4ec400a9d633311b2d0bec2661422e24bdb135ec3a6c5b4e2e522a3", + "prompt_sha256": "9484ef8602be5a89d2d1e6920d3f8bb155458f4c052e1dbedc3971c6628801ee", + "source_sha256": "bc410ed908e0cd7bb2ecf791eb66f9e8d6a83c401bf85e50485e510667fedf77", + "output_sha256": "cfd11b999d0f79757a03811f9de79c27e99227af6a863ede9080c9c1967526c8", + "repo_head": "c067989ac5b9c5e5d5dc01550daf77e165d790ef", + "runtime_policy_sha256": "35abc2c6bc001092dc4899b7cea6acf0065e854307a7acde80c18b9f78d8cc2c" + }, + "lease": { + "initial_probe": { + "schema": "ltx.benchmark.lock-probe.v1", + "lease_path": "/Users//Library/Application Support/LTX Shared/local-metal.lock", + "observed": "acquired", + "holder_payload": { + "acquired_at_utc": "2026-07-27T00:47:23.640302+00:00", + "host": ".local", + "job_id": "920d95d0", + "pid": 40997, + "ppid": 40991, + "product": "LTX Desktop", + "reason": "mlx local video generation", + "schema": "ltx.local-metal-lock.v1", + "workload": "fast_t2v" + }, + "holder_payload_schema_valid": true, + "timestamp": 1785113351.733391, + "monotonic_ns": 345457663949291 + }, + "running_probe": { + "schema": "ltx.benchmark.lock-probe.v1", + "lease_path": "/Users//Library/Application Support/LTX Shared/local-metal.lock", + "observed": "contended", + "holder_payload": { + "acquired_at_utc": "2026-07-27T00:49:11.735767+00:00", + "host": ".local", + "job_id": "25f4bcac", + "pid": 57580, + "ppid": 57574, + "product": "LTX Desktop", + "reason": "mlx local video generation", + "schema": "ltx.local-metal-lock.v1", + "workload": "fast_t2v" + }, + "holder_payload_schema_valid": true, + "timestamp": 1785113351.737553, + "monotonic_ns": 345457668111541 + }, + "terminal_probe": { + "schema": "ltx.benchmark.lock-probe.v1", + "lease_path": "/Users//Library/Application Support/LTX Shared/local-metal.lock", + "observed": "acquired", + "holder_payload": { + "acquired_at_utc": "2026-07-27T00:49:11.735767+00:00", + "host": ".local", + "job_id": "25f4bcac", + "pid": 57580, + "ppid": 57574, + "product": "LTX Desktop", + "reason": "mlx local video generation", + "schema": "ltx.local-metal-lock.v1", + "workload": "fast_t2v" + }, + "holder_payload_schema_valid": true, + "timestamp": 1785113427.47246, + "monotonic_ns": 345533402294708 + } + }, + "runtime_summary": { + "samples": 73, + "peak_rss_gib": 0.486328125, + "peak_mlx_mib": 38679, + "peak_mps_allocated_mib": 0, + "peak_mps_driver_mib": 8, + "peak_process_tree_rss_gib": 35.02256774902344, + "peak_physical_footprint_gib": 41.33585896342993, + "peak_cpu_percent": 381.9, + "peak_gpu_utilization_percent": 100.0, + "mlx_profile_status": "success", + "mlx_profile_phase": "Decoding video + audio + muxing", + "mlx_profile_path": "/Users//Library/Application Support/LTXDesktop/outputs/.mlx-profiles/ltx2_video_20260726_184911_64fafc87-6b47ce0baed54185a1ded73aa5e75724.jsonl", + "mlx_profile_sampled_at": "2026-07-27T00:50:26.615068+00:00", + "mlx_runtime_identity": { + "runtime_commit": "3171bac4ba901c0237faea2678c34034b37abc2a", + "runtime_dirty": false, + "runtime_version": "0.14.20.dev1", + "core_version": "0.14.20.dev1", + "mlx_version": "0.31.1", + "mlx_metal_version": "0.31.1", + "device_name": "Apple M5 Max", + "device_architecture": "applegpu_g17s", + "device_memory_bytes": 137438953472, + "device_recommended_working_set_bytes": 115448725504, + "runtime_family": "0.14", + "device_family": "applegpu_g17s" + } + }, + "machine": { + "captured_at_utc": "2026-07-27T00:50:27.513365+00:00", + "host": ".local", + "python": "3.13.12 (main, Mar 25 2026, 02:48:22) [Clang 22.1.1 ]", + "sw_vers": "ProductName:\t\tmacOS\nProductVersion:\t\t26.5.2\nBuildVersion:\t\t25F84", + "hardware": "Hardware:\n\n Hardware Overview:\n\n Model Name: MacBook Pro\n Model Identifier: Mac17,6\n Model Number: Z1N20001GLL/A\n Chip: Apple M5 Max\n Total Number of Cores: 18 (6 Super and 12 Performance)\n Memory: 128 GB\n System Firmware Version: 18000.121.3\n OS Loader Version: 18000.121.3", + "memsize": "137438953472", + "repo_head": "c067989ac5b9c5e5d5dc01550daf77e165d790ef", + "repo_status": "(local working tree; not part of this change)", + "ffmpeg": [ + "ffmpeg version 8.1.1 Copyright (c) 2000-2026 the FFmpeg developers" + ] + }, + "cancel_sent": false, + "strict_failures": [], + "strict_status": "PASS" +} \ No newline at end of file diff --git a/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_profiled/plan.json b/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_profiled/plan.json new file mode 100644 index 000000000..d28ef10a6 --- /dev/null +++ b/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_profiled/plan.json @@ -0,0 +1,24 @@ +{ + "schema": "ltx.hd-benchmark.plan.v1", + "product": "ltx_desktop_electron", + "production_entrypoint": "POST /api/generate", + "production_lease": "/Users//Library/Application Support/LTX Shared/local-metal.lock", + "expect_mode": "eager", + "runs_per_case": 1, + "cases": [ + { + "case_id": "fast_t2v_540p_5s", + "resolution": "540p", + "duration": 5, + "fps": 24, + "aspect_ratio": "16:9", + "image_conditioned": false + } + ], + "notes": [ + "Run once with --expect-mode=eager and once with --expect-mode=low_ram; runtime policy, not a request-only toggle, owns the app mode.", + "Distilled Fast T2V/I2V cannot use TeaCache and this harness never presents it as a switch.", + "Explicit modality-tiling candidates are qualified in the a second local application MLX HQ matrix; Electron Fast exposes only its production automatic policy.", + "The production flock serializes local heavy work; cloud/API/CPU-only work remains outside it." + ] +} \ No newline at end of file diff --git a/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_profiled/summary.md b/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_profiled/summary.md new file mode 100644 index 000000000..a238ea2f2 --- /dev/null +++ b/backend/performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_profiled/summary.md @@ -0,0 +1,7 @@ +# LTX Desktop Electron HD benchmark + +Command: `/Users//projects/ltx-desktop-electron/backend/performance_runner/hd_matrix.py --execute --expect-mode=eager --case fast_t2v_540p_5s --artifacts performance_runner/artifacts/production_20260726/electron_auto_mlx_540p_profiled` + +| case | mode | status | wall s | footprint GiB | MLX MiB | SHA-256 | strict gaps | +|---|---|---:|---:|---:|---:|---|---| +| fast_t2v_540p_5s | eager | PASS | 76.01 | 41.33585896342993 | 38679 | cfd11b999d0f79757a03811f9de79c27e99227af6a863ede9080c9c1967526c8 | — | diff --git a/backend/performance_runner/artifacts/production_20260726/qualification_report.md b/backend/performance_runner/artifacts/production_20260726/qualification_report.md new file mode 100644 index 000000000..713d45b06 --- /dev/null +++ b/backend/performance_runner/artifacts/production_20260726/qualification_report.md @@ -0,0 +1,48 @@ +# LTX Desktop Electron production qualification — 2026-07-26 + +Status: **PASS after dimension normalization fix `f539eb6`** + +## Accepted production run + +The final fixed-seed `fast_t2v_540p_5s` run used `POST /api/generate`, AUTO→MLX, BF16 eager mode, no tiling, and exact clean sidecar runtime `0.14.20.dev1` at `3171bac4ba901c0237faea2678c34034b37abc2a` with MLX 0.31.1. + +| evidence | result | +|---|---:| +| completion-resolved dimensions | 896×512 | +| sidecar command dimensions | 896×512 | +| ffprobe dimensions | 896×512 | +| frames / duration | 121 / 5.0417 s | +| wall time | 66.608 s | +| peak process-tree RSS | 34.136 GiB | +| peak physical footprint | 41.133 GiB | +| peak MLX allocation | 38,585 MiB | +| peak GPU utilization sample | 100% | +| output SHA-256 | `5f1f02369e48d61b2951bc4d5b38321d4e6d22c65dee2e4870d6ee20de15185c` | + +Strict failures are empty. The terminal runtime sample reports profile status `success`, active MLX memory 0 MiB, inactive allocator cache 2 MiB, MPS allocation 0 MiB, no active lease owner, and a free production flock. `useLocalTextEncoder` was restored to its original `false` value and the QA backend was stopped after the run. + +## Pre-fix versus fixed run + +| run | reported resolved | ffprobe actual | wall | physical | MLX peak | qualification | +|---|---|---|---:|---:|---:|---| +| pre-fix `c067989` | 960×544 | 960×512 | 76.013 s | 41.336 GiB | 38,679 MiB | reject: silent dimension mismatch | +| fixed `f539eb6` | 896×512 | 896×512 | 66.608 s | 41.133 GiB | 38,585 MiB | PASS | + +The fixed run is 9.405 s faster, but it also renders 6.7% fewer pixels per frame; this is not treated as an optimization speedup claim. The important qualification change is that API, command, and media dimensions agree exactly on a valid distilled 64-pixel grid. The pre-fix result is retained as regression evidence even though its old harness status was PASS. + +## Historical comparison + +The preserved pre-change evidence is a direct-engine baseline, not an Electron app-entrypoint result. Its eager 704×448×96 run took 36.45 s with 35.20 GiB RSS and 38.81 GiB physical footprint; its eager 1280×704×96 run took 129.87 s with 35.20 GiB RSS and 45.42 GiB physical footprint. The final production run is about 1.20 seconds per million actual pixel-frames, comparable to the small historical run and below the historical HD value of about 1.50. Shape, frame count, entrypoint, and runtime orchestration differ, so these normalized figures are context rather than an A/B performance claim. + +## Capability fallback, lease, and crash gates + +- Runtime policy maps prepared text embeddings, A2V, Retake, Extend, IC-LoRA, and image generation to Torch. The interop smoke completed Electron job `de159ca4` through the prepared-embedding path while a second local application waited for the shared lease. Its compact artifact proves completion and ordering, but does not persist a selected-engine field; it is routing/interop evidence, not a strict Torch performance measurement. +- Cross-product lease test: PASS. Electron owned first, a second local application acquired only after Electron release, CPU-only control reads remained responsive, and the terminal flock was free. +- Crash release: PASS. Kernel flock authority released even when stale diagnostic JSON remained. +- TeaCache and explicit tiling: not applicable to Electron Fast distilled T2V/I2V. Dev/HQ TeaCache and modality-tiling candidates are covered only by the a second local application matrix. + +## Scoped Metal trace + +The direct-engine Metal System Trace completed with target exit 0 and exact runtime identity. After PID filtering it contained 22,159 target GPU intervals; 22,149 were compute. Median/p95/p99/max interval durations were 1.384/7.951/9.122/33.928 ms. Shader Timeline was disabled, so the report makes command-buffer/encoder hotspot claims only, not kernel-symbol claims. The raw trace and large XML exports were not committed. + +Compact strict results, interop/crash proofs, trace manifest/hotspots, and reports are retained. Raw process/progress/runtime samples were pruned after deriving the evidence above. diff --git a/backend/performance_runner/artifacts/production_20260726/xctrace_direct_engine/hotspots.json b/backend/performance_runner/artifacts/production_20260726/xctrace_direct_engine/hotspots.json new file mode 100644 index 000000000..4fa4ad414 --- /dev/null +++ b/backend/performance_runner/artifacts/production_20260726/xctrace_direct_engine/hotspots.json @@ -0,0 +1,391 @@ +{ + "schema": "ltx.metal-hotspot-report.v1", + "source_export": "/Users//projects/ltx-desktop-electron/backend/performance_runner/artifacts/production_20260726/xctrace_direct_engine/gpu_intervals.xml", + "target_pid": 54125, + "interval_count": 22159, + "trace_window_seconds": 56.438584375, + "sum_interval_seconds": 52.300371667, + "duration_ms": { + "mean": 2.360231583871113, + "median": 1.38375, + "p95": 7.9506712, + "p99": 9.122231279999962, + "max": 33.928375 + }, + "channels": [ + { + "channel": "Compute", + "count": 22149, + "sum_interval_seconds": 52.298434959 + }, + { + "channel": "Fragment", + "count": 10, + "sum_interval_seconds": 0.001936708 + } + ], + "label_groups": [ + { + "label": "Command Buffer 0:Compute Command 0", + "count": 20804, + "sum_interval_seconds": 51.280439953 + }, + { + "label": "Command Buffer 0:Compute Command 1", + "count": 350, + "sum_interval_seconds": 0.448542333 + }, + { + "label": "Command Buffer 0:Compute Command 2", + "count": 280, + "sum_interval_seconds": 0.329260706 + }, + { + "label": "Command Buffer 0:Compute Command 5", + "count": 131, + "sum_interval_seconds": 0.086180545 + }, + { + "label": "Command Buffer 0:Compute Command 3", + "count": 211, + "sum_interval_seconds": 0.057650971 + }, + { + "label": "Command Buffer 0:Compute Command 4", + "count": 154, + "sum_interval_seconds": 0.03583012 + }, + { + "label": "Command Buffer 0:Compute Command 8", + "count": 55, + "sum_interval_seconds": 0.030191043 + }, + { + "label": "Command Buffer 0:Compute Command 6", + "count": 103, + "sum_interval_seconds": 0.025114327 + }, + { + "label": "Command Buffer 0:Compute Command 7", + "count": 62, + "sum_interval_seconds": 0.003280459 + }, + { + "label": "Command Buffer 0:Compute Command 10", + "count": 4, + "sum_interval_seconds": 0.002637875 + }, + { + "label": "Command Buffer 0:Compute Command 9", + "count": 5, + "sum_interval_seconds": 0.001243335 + } + ], + "top_dispatches": [ + { + "start_ns": 58478020416, + "duration_ns": 33928375, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x40264c4", + "command_buffer_id": "0x60301691c", + "encoder_id": "0x60301691d", + "submission_id": "67,265,732", + "start_seconds": 58.478020416, + "duration_ms": 33.928375 + }, + { + "start_ns": 3845270041, + "duration_ns": 32951792, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x400b7d2", + "command_buffer_id": "0x60300a907", + "encoder_id": "0x60300a908", + "submission_id": "67,155,922", + "start_seconds": 3.845270041, + "duration_ms": 32.951792 + }, + { + "start_ns": 58085982125, + "duration_ns": 25966833, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x4026371", + "command_buffer_id": "0x6030168fb", + "encoder_id": "0x6030168fc", + "submission_id": "67,265,393", + "start_seconds": 58.085982125, + "duration_ms": 25.966833 + }, + { + "start_ns": 57886002208, + "duration_ns": 25946542, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x40262c2", + "command_buffer_id": "0x6030168e9", + "encoder_id": "0x6030168ea", + "submission_id": "67,265,218", + "start_seconds": 57.886002208, + "duration_ms": 25.946542 + }, + { + "start_ns": 58886018916, + "duration_ns": 25928792, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x402662c", + "command_buffer_id": "0x60301693e", + "encoder_id": "0x60301693f", + "submission_id": "67,266,092", + "start_seconds": 58.886018916, + "duration_ms": 25.928792 + }, + { + "start_ns": 58686031666, + "duration_ns": 25917875, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x4026573", + "command_buffer_id": "0x60301692c", + "encoder_id": "0x60301692d", + "submission_id": "67,265,907", + "start_seconds": 58.686031666, + "duration_ms": 25.917875 + }, + { + "start_ns": 57685999875, + "duration_ns": 25149000, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x4026124", + "command_buffer_id": "0x6030168da", + "encoder_id": "0x6030168db", + "submission_id": "67,264,804", + "start_seconds": 57.685999875, + "duration_ms": 25.149 + }, + { + "start_ns": 42390477958, + "duration_ns": 24413583, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x4020a7a", + "command_buffer_id": "0x603014bac", + "encoder_id": "0x603014bad", + "submission_id": "67,242,618", + "start_seconds": 42.390477958, + "duration_ms": 24.413583 + }, + { + "start_ns": 35279558166, + "duration_ns": 23188125, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x401df6a", + "command_buffer_id": "0x60301352c", + "encoder_id": "0x60301352d", + "submission_id": "67,231,594", + "start_seconds": 35.279558166, + "duration_ms": 23.188125 + }, + { + "start_ns": 35999744458, + "duration_ns": 23054333, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x401e3c6", + "command_buffer_id": "0x603013773", + "encoder_id": "0x603013774", + "submission_id": "67,232,710", + "start_seconds": 35.999744458, + "duration_ms": 23.054333 + }, + { + "start_ns": 23460940500, + "duration_ns": 21874500, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x4019517", + "command_buffer_id": "0x603010e0b", + "encoder_id": "0x603010e0c", + "submission_id": "67,212,567", + "start_seconds": 23.4609405, + "duration_ms": 21.8745 + }, + { + "start_ns": 27052850500, + "duration_ns": 20222333, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x401ab6d", + "command_buffer_id": "0x603011a16", + "encoder_id": "0x603011a17", + "submission_id": "67,218,285", + "start_seconds": 27.0528505, + "duration_ms": 20.222333 + }, + { + "start_ns": 4187496916, + "duration_ns": 20132792, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 1 ( python3.11 (54125) ) 0x400bb28", + "command_buffer_id": "0x60300a9bc", + "encoder_id": "0x60300a9be", + "submission_id": "67,156,776", + "start_seconds": 4.187496916, + "duration_ms": 20.132792 + }, + { + "start_ns": 3934889833, + "duration_ns": 18933583, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 1 ( python3.11 (54125) ) 0x400b89c", + "command_buffer_id": "0x60300a913", + "encoder_id": "0x60300a915", + "submission_id": "67,156,124", + "start_seconds": 3.934889833, + "duration_ms": 18.933583 + }, + { + "start_ns": 44276251000, + "duration_ns": 18729458, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x40215ab", + "command_buffer_id": "0x603015168", + "encoder_id": "0x603015169", + "submission_id": "67,245,483", + "start_seconds": 44.276251, + "duration_ms": 18.729458 + }, + { + "start_ns": 32127840416, + "duration_ns": 18157250, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x401cad7", + "command_buffer_id": "0x603012adb", + "encoder_id": "0x603012adc", + "submission_id": "67,226,327", + "start_seconds": 32.127840416, + "duration_ms": 18.15725 + }, + { + "start_ns": 59093994666, + "duration_ns": 17954959, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x40266d2", + "command_buffer_id": "0x60301694d", + "encoder_id": "0x60301694e", + "submission_id": "67,266,258", + "start_seconds": 59.093994666, + "duration_ms": 17.954959 + }, + { + "start_ns": 59294006208, + "duration_ns": 17941500, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x4026781", + "command_buffer_id": "0x60301695f", + "encoder_id": "0x603016960", + "submission_id": "67,266,433", + "start_seconds": 59.294006208, + "duration_ms": 17.9415 + }, + { + "start_ns": 27744204625, + "duration_ns": 17670333, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x401afb7", + "command_buffer_id": "0x603011c5d", + "encoder_id": "0x603011c5e", + "submission_id": "67,219,383", + "start_seconds": 27.744204625, + "duration_ms": 17.670333 + }, + { + "start_ns": 39227983875, + "duration_ns": 17573666, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x401f769", + "command_buffer_id": "0x6030141c4", + "encoder_id": "0x6030141c5", + "submission_id": "67,237,737", + "start_seconds": 39.227983875, + "duration_ms": 17.573666 + }, + { + "start_ns": 29659806500, + "duration_ns": 17274083, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x401bb7e", + "command_buffer_id": "0x6030122ad", + "encoder_id": "0x6030122ae", + "submission_id": "67,222,398", + "start_seconds": 29.6598065, + "duration_ms": 17.274083 + }, + { + "start_ns": 35103428250, + "duration_ns": 16151666, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x401de4d", + "command_buffer_id": "0x603013498", + "encoder_id": "0x603013499", + "submission_id": "67,231,309", + "start_seconds": 35.10342825, + "duration_ms": 16.151666 + }, + { + "start_ns": 41278970458, + "duration_ns": 15950083, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x40203d7", + "command_buffer_id": "0x60301483e", + "encoder_id": "0x60301483f", + "submission_id": "67,240,919", + "start_seconds": 41.278970458, + "duration_ms": 15.950083 + }, + { + "start_ns": 29304280416, + "duration_ns": 15439292, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x401b959", + "command_buffer_id": "0x603012186", + "encoder_id": "0x603012187", + "submission_id": "67,221,849", + "start_seconds": 29.304280416, + "duration_ms": 15.439292 + }, + { + "start_ns": 38152304583, + "duration_ns": 15408500, + "channel": "Compute", + "depth": 0, + "label": "Command Buffer 0:Compute Command 0 ( python3.11 (54125) ) 0x401f0d4", + "command_buffer_id": "0x603013e56", + "encoder_id": "0x603013e57", + "submission_id": "67,236,052", + "start_seconds": 38.152304583, + "duration_ms": 15.4085 + } + ], + "limitations": [ + "Metal System Trace recorded Shader Timeline: Disabled, so kernel symbols and per-shader counters are unavailable.", + "GPU intervals can be nested; summed interval duration is not GPU wall time or utilization.", + "Rows are filtered to the target child PID so unrelated system Metal activity is excluded." + ] +} diff --git a/backend/performance_runner/artifacts/production_20260726/xctrace_direct_engine/manifest_rerun.json b/backend/performance_runner/artifacts/production_20260726/xctrace_direct_engine/manifest_rerun.json new file mode 100644 index 000000000..0b0288375 --- /dev/null +++ b/backend/performance_runner/artifacts/production_20260726/xctrace_direct_engine/manifest_rerun.json @@ -0,0 +1,57 @@ +{ + "ended_unix_seconds": 1785112940.454589, + "export_returncode": 0, + "forwarded_environment_names": [ + "LTX2_MEDIA_WRITE_OVERLAP" + ], + "record_returncode": 0, + "runtime": { + "core_version": "0.14.20.dev1", + "device_architecture": "applegpu_g17s", + "device_memory_bytes": 137438953472, + "device_name": "Apple M5 Max", + "device_recommended_working_set_bytes": 115448725504, + "mlx_metal_version": "0.31.1", + "mlx_version": "0.31.1", + "runtime_commit": "3171bac4ba901c0237faea2678c34034b37abc2a", + "runtime_dirty": false, + "runtime_version": "0.14.20.dev1" + }, + "schema": "ltx.metal-system-trace.v1", + "started_unix_seconds": 1785112525.8482711, + "target_command": [ + "/usr/bin/python3", + "-c", + "import fcntl,pathlib,subprocess,sys; p=pathlib.Path.home()/\"Library/Application Support/LTX Shared/local-metal.lock\"; p.parent.mkdir(parents=True,exist_ok=True); f=p.open(\"a+\"); fcntl.flock(f.fileno(),fcntl.LOCK_EX); raise SystemExit(subprocess.call(sys.argv[1:]))", + "/Users//video-models/ltx-2-mlx/.venv/bin/python", + "-m", + "ltx_pipelines_mlx.cli", + "generate", + "--prompt", + "A slow, steady lateral camera move reveals the scene with natural subject motion and physically consistent lighting.", + "--distilled", + "--model", + "dgrauet/ltx-2.3-mlx", + "--model-precision", + "bf16", + "--seed", + "424242", + "--height", + "512", + "--width", + "768", + "--frames", + "129", + "--frame-rate", + "24", + "--profile-json", + "/tmp/ltx_qa_20260726_profile_rerun.jsonl", + "--output", + "/tmp/ltx_qa_20260726_rerun.mp4" + ], + "template": "Metal System Trace", + "toc_path": "/private/tmp/ltx_qa_20260726_rerun.trace.toc.xml", + "toc_sha256": "44f287734d6766948606ef0cfa158fcc2260b2bb3b5afb4731e3b646a7047bfc", + "trace_path": "/private/tmp/ltx_qa_20260726_rerun.trace", + "trace_sha256": null +} diff --git a/backend/performance_runner/artifacts/production_20260726/xctrace_direct_engine/report.md b/backend/performance_runner/artifacts/production_20260726/xctrace_direct_engine/report.md new file mode 100644 index 000000000..e89173bdf --- /dev/null +++ b/backend/performance_runner/artifacts/production_20260726/xctrace_direct_engine/report.md @@ -0,0 +1,42 @@ +# Scoped Metal System Trace qualification + +Status: **PASS with symbol-level limitation** + +The recorder launched one fixed-seed direct-engine workload under the `Metal System Trace` template while a wrapper held the production shared flock for the full subprocess lifetime. The target exited 0; xctrace record and TOC export both exited 0. The production Electron and a second local application entrypoints were idle for the capture. + +## Workload and identity + +- Runtime: `ltx-pipelines-mlx 0.14.20.dev1` at clean revision `3171bac4ba901c0237faea2678c34034b37abc2a` +- Device: Apple M5 Max, 128 GiB unified memory, MLX 0.31.1 +- Model: `dgrauet/ltx-2.3-mlx`, BF16, distilled two-stage +- Shape: 768×512×129 at 24 fps; seed 424242 +- Target child: `python3.11` PID 54125, exit 0 +- Engine-reported render time: 59.0 s +- Target GPU interval window: 56.439 s + +## Per-dispatch hotspot summary + +The focused export contains 22,159 target-process GPU intervals after excluding Codex, loginwindow, and other system Metal traffic. Compute accounts for 22,149 intervals and 52.298 s of the 52.300 s summed interval duration. Interval durations are nested and therefore are not additive GPU wall time. + +| statistic | duration | +|---|---:| +| median | 1.384 ms | +| p95 | 7.951 ms | +| p99 | 9.122 ms | +| maximum | 33.928 ms | + +The slowest recorded intervals were all compute encoders: + +| trace start | duration | command buffer | encoder | submission | +|---:|---:|---|---|---:| +| 58.478 s | 33.928 ms | `0x60301691c` | `0x60301691d` | 67,265,732 | +| 3.845 s | 32.952 ms | `0x60300a907` | `0x60300a908` | 67,155,922 | +| 58.086 s | 25.967 ms | `0x6030168fb` | `0x6030168fc` | 67,265,393 | +| 57.886 s | 25.947 ms | `0x6030168e9` | `0x6030168ea` | 67,265,218 | +| 58.886 s | 25.929 ms | `0x60301693e` | `0x60301693f` | 67,266,092 | + +`Compute Command 0` dominates the generic encoder labels (20,804 intervals; 51.280 summed seconds). The template recorded `Shader Timeline: Disabled`, so this capture cannot attribute those dispatches to kernel symbols or shader-counter metrics. The command-buffer/encoder IDs above are the most precise per-dispatch attribution supported by this capture; kernel-name claims would be unsupported. + +## Artifact policy + +`manifest_rerun.json`, `hotspots.json`, and this report are the compact committed evidence. The raw trace bundle remains at `/private/tmp/ltx_qa_20260726_rerun.trace`; focused XML exports are intentionally uncommitted because they are large derived data. The manifest does not contain a raw trace SHA because an `.trace` is a directory bundle rather than a regular file. diff --git a/backend/performance_runner/baselines/ltx_direct_engine_2026-07-26.json b/backend/performance_runner/baselines/ltx_direct_engine_2026-07-26.json new file mode 100644 index 000000000..dbbccc2e7 --- /dev/null +++ b/backend/performance_runner/baselines/ltx_direct_engine_2026-07-26.json @@ -0,0 +1,13 @@ +{ + "schema": "ltx.hd-benchmark.external-baseline.v1", + "qualification": "Historical direct-engine baseline; not an Electron app-entrypoint result. Use only to reconstruct pre-change expectations.", + "source_repo": "/Users//projects/ltx-desktop", + "source_entrypoint": "backend/engine/generate_v23.py", + "runs": [ + {"requested": {"width": 704, "height": 480, "frames": 97}, "actual": {"width": 704, "height": 448, "frames": 96}, "mode": "eager", "wall_seconds": 36.45, "peak_rss_gib": 35.20, "peak_physical_gib": 38.81}, + {"requested": {"width": 704, "height": 480, "frames": 97}, "actual": {"width": 704, "height": 448, "frames": 96}, "mode": "low_ram", "wall_seconds": 36.54, "peak_rss_gib": 15.45, "peak_physical_gib": 23.03, "byte_identical_to_eager": true}, + {"requested": {"width": 1280, "height": 704, "frames": 97}, "actual": {"width": 1280, "height": 704, "frames": 96}, "mode": "eager", "wall_seconds": 129.87, "peak_rss_gib": 35.20, "peak_physical_gib": 45.42}, + {"requested": {"width": 1280, "height": 704, "frames": 97}, "actual": {"width": 1280, "height": 704, "frames": 96}, "mode": "low_ram", "wall_seconds": 129.35, "peak_rss_gib": 15.45, "peak_physical_gib": 45.61}, + {"requested": {"width": 1280, "height": 704, "frames": 97}, "actual": {"width": 1280, "height": 704, "frames": 96}, "mode": "low_ram_vae_0_5", "wall_seconds": 133.66, "peak_rss_gib": 15.45, "peak_physical_gib": 34.316, "psnr_db": 49.2577, "ssim": 0.993787} + ] +} diff --git a/backend/performance_runner/hd_matrix.py b/backend/performance_runner/hd_matrix.py new file mode 100644 index 000000000..a2a0ab67c --- /dev/null +++ b/backend/performance_runner/hd_matrix.py @@ -0,0 +1,542 @@ +#!/usr/bin/env python3 +"""HD qualification through the production LTX Desktop FastAPI entrypoint. + +The default action prints a no-render plan. Use ``--execute`` only after the +backend/app is running and local Metal qualification is authorized. +""" +from __future__ import annotations + +import argparse +import array +import concurrent.futures +import ctypes +import fcntl +import hashlib +import json +import math +import os +import re +import socket +import subprocess +import sys +import time +import urllib.error +import urllib.request +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +try: + from . import perf_config +except ImportError: # direct `python hd_matrix.py` execution + import perf_config + +LOCK_PATH = Path.home() / "Library/Application Support/LTX Shared/local-metal.lock" +LOCK_SCHEMA = "ltx.local-metal-lock.v1" +PROMPT = "A slow, steady lateral camera move reveals the scene with natural subject motion and physically consistent lighting." +TERMINAL = {"complete", "cancelled", "error"} + + +@dataclass(frozen=True) +class MatrixCase: + case_id: str + resolution: str + duration: int + fps: int = 24 + aspect_ratio: str = "16:9" + image_conditioned: bool = False + + def payload(self, image_path: str | None) -> dict[str, Any]: + return { + "prompt": PROMPT, + "resolution": self.resolution, + "model": "fast", + "cameraMotion": "none", + "negativePrompt": "", + "duration": self.duration, + "fps": self.fps, + "audio": False, + "imagePath": image_path if self.image_conditioned else None, + "audioPath": None, + "aspectRatio": self.aspect_ratio, + "seed": 424242, + "loras": [], + } + + +def default_matrix() -> list[MatrixCase]: + return [ + MatrixCase("fast_t2v_540p_5s", "540p", 5), + MatrixCase("fast_t2v_720p_5s", "720p", 5), + MatrixCase("fast_t2v_720p_8s", "720p", 8), + MatrixCase("fast_t2v_1080p_5s", "1080p", 5), + MatrixCase("fast_i2v_720p_5s", "720p", 5, image_conditioned=True), + ] + + +def canonical_sha(value: Any) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode()).hexdigest() + + +def file_sha(path: str | Path | None) -> str | None: + if not path or not Path(path).is_file(): + return None + digest = hashlib.sha256() + with open(path, "rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def http_json(method: str, path: str, body: dict[str, Any] | None = None, timeout: float = 1800) -> Any: + data = None if body is None else json.dumps(body).encode() + req = urllib.request.Request( + perf_config.BASE_URL + path, + data=data, + method=method, + headers={"Content-Type": "application/json", **perf_config._auth_headers()}, + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as response: + raw = response.read().decode() + except urllib.error.HTTPError as exc: + detail = exc.read().decode(errors="replace") + raise RuntimeError(f"{method} {path} -> HTTP {exc.code}: {detail}") from exc + return json.loads(raw) if raw else None + + +def probe_metal_lock(path: Path = LOCK_PATH) -> dict[str, Any]: + path.parent.mkdir(parents=True, exist_ok=True) + payload: Any = None + observed = "contended" + with open(path, "a+", encoding="utf-8") as stream: + stream.seek(0) + raw = stream.read().strip() + if raw: + try: + payload = json.loads(raw) + except json.JSONDecodeError: + payload = {"unparsed": raw[:1000]} + try: + fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + observed = "acquired" + fcntl.flock(stream.fileno(), fcntl.LOCK_UN) + except BlockingIOError: + observed = "contended" + return { + "schema": "ltx.benchmark.lock-probe.v1", + "lease_path": str(path), + "observed": observed, + "holder_payload": payload, + "holder_payload_schema_valid": isinstance(payload, dict) and payload.get("schema") == LOCK_SCHEMA, + "timestamp": time.time(), + "monotonic_ns": time.monotonic_ns(), + } + + +def _ps_rows() -> list[tuple[int, int, int, float]]: + try: + text = subprocess.check_output(["ps", "-A", "-o", "pid=", "-o", "ppid=", "-o", "rss=", "-o", "%cpu="], text=True, timeout=5) + except (OSError, subprocess.SubprocessError): + return [] + rows = [] + for line in text.splitlines(): + parts = line.split() + if len(parts) == 4: + try: + rows.append((int(parts[0]), int(parts[1]), int(parts[2]), float(parts[3]))) + except ValueError: + pass + return rows + + +def descendant_pids(root_pid: int, rows: list[tuple[int, int, int, float]]) -> set[int]: + selected = {root_pid} + changed = True + while changed: + changed = False + for pid, ppid, _rss, _cpu in rows: + if ppid in selected and pid not in selected: + selected.add(pid) + changed = True + return selected + + +class _RusageInfoV2(ctypes.Structure): + _fields_ = [("ri_uuid", ctypes.c_uint8 * 16)] + [ + (name, ctypes.c_uint64) for name in ( + "ri_user_time", "ri_system_time", "ri_pkg_idle_wkups", "ri_interrupt_wkups", + "ri_pageins", "ri_wired_size", "ri_resident_size", "ri_phys_footprint", + "ri_proc_start_abstime", "ri_proc_exit_abstime", "ri_child_user_time", + "ri_child_system_time", "ri_child_pkg_idle_wkups", "ri_child_interrupt_wkups", + "ri_child_pageins", "ri_child_elapsed_abstime", "ri_diskio_bytesread", + "ri_diskio_byteswritten", + ) + ] + + +def physical_footprint_bytes(pid: int) -> int | None: + if sys.platform != "darwin": + return None + try: + libproc = ctypes.CDLL("/usr/lib/libproc.dylib") + fn = libproc.proc_pid_rusage + fn.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_void_p] + info = _RusageInfoV2() + if fn(pid, 2, ctypes.byref(info)) == 0: + return int(info.ri_phys_footprint) + except (OSError, AttributeError): + pass + return None + + +def gpu_utilization_percent() -> float | None: + if sys.platform != "darwin": + return None + try: + text = subprocess.check_output(["ioreg", "-r", "-d", "1", "-w", "0", "-c", "IOAccelerator"], text=True, timeout=5) + except (OSError, subprocess.SubprocessError): + return None + values: list[float] = [] + for pattern in (r'"Device Utilization %"\s*=\s*(\d+)', r'"GPU Activity\(%\)"\s*=\s*(\d+)'): + values.extend(float(raw) for raw in re.findall(pattern, text)) + return max(values) if values else None + + +def process_sample(root_pid: int) -> dict[str, Any]: + rows = _ps_rows() + pids = descendant_pids(root_pid, rows) + chosen = [row for row in rows if row[0] in pids] + footprints = [value for value in (physical_footprint_bytes(pid) for pid in pids) if value is not None] + return { + "timestamp": time.time(), + "root_pid": root_pid, + "pids": sorted(pids), + "process_count": len(chosen), + "rss_gib": sum(row[2] for row in chosen) * 1024 / 1024**3, + "physical_footprint_gib": sum(footprints) / 1024**3 if footprints else None, + "cpu_percent": sum(row[3] for row in chosen), + "gpu_utilization_percent": gpu_utilization_percent(), + } + + +def ffprobe(path: str) -> dict[str, Any]: + raw = subprocess.check_output(["ffprobe", "-v", "error", "-print_format", "json", "-show_format", "-show_streams", path], text=True, timeout=30) + data = json.loads(raw) + streams = data.get("streams") or [] + video = next((row for row in streams if row.get("codec_type") == "video"), {}) + audio = next((row for row in streams if row.get("codec_type") == "audio"), None) + duration = video.get("duration") or (data.get("format") or {}).get("duration") + return { + "width": video.get("width"), "height": video.get("height"), + "frames": int(video["nb_frames"]) if str(video.get("nb_frames", "")).isdigit() else None, + "duration_seconds": float(duration) if duration is not None else None, + "has_audio": audio is not None, + "video_codec": video.get("codec_name"), "audio_codec": audio.get("codec_name") if audio else None, + } + + +def _ffmpeg_metric(reference: str, candidate: str, filter_name: str, pattern: str) -> float | None: + proc = subprocess.run(["ffmpeg", "-v", "info", "-i", reference, "-i", candidate, "-lavfi", filter_name, "-f", "null", "-"], capture_output=True, text=True, timeout=1800) + matches = re.findall(pattern, proc.stderr) + return float(matches[-1]) if matches else None + + +def _audio_pcm(path: str) -> array.array: + proc = subprocess.run(["ffmpeg", "-v", "error", "-i", path, "-map", "0:a:0?", "-ac", "1", "-ar", "48000", "-f", "f32le", "-"], capture_output=True, timeout=600) + values = array.array("f") + values.frombytes(proc.stdout) + return values + + +def quality_metrics(reference: str, candidate: str) -> dict[str, Any]: + result: dict[str, Any] = { + "psnr_db": _ffmpeg_metric(reference, candidate, "psnr", r"average:([0-9.]+)"), + "ssim": _ffmpeg_metric(reference, candidate, "ssim", r"All:([0-9.]+)"), + "audio_mae": None, "audio_snr_db": None, + } + a, b = _audio_pcm(reference), _audio_pcm(candidate) + count = min(len(a), len(b)) + if count: + signal = sum(float(a[i]) ** 2 for i in range(count)) / count + noise = sum((float(a[i]) - float(b[i])) ** 2 for i in range(count)) / count + result["audio_mae"] = sum(abs(float(a[i]) - float(b[i])) for i in range(count)) / count + result["audio_snr_db"] = math.inf if noise == 0 else 10.0 * math.log10(signal / noise) if signal else None + return result + + +def machine_details(repo_root: Path) -> dict[str, Any]: + def text(argv: list[str]) -> str: + try: + return subprocess.check_output(argv, text=True, stderr=subprocess.DEVNULL, timeout=15).strip() + except (OSError, subprocess.SubprocessError): + return "" + return { + "captured_at_utc": datetime.now(timezone.utc).isoformat(), "host": socket.gethostname(), "python": sys.version, + "sw_vers": text(["sw_vers"]), "hardware": text(["system_profiler", "SPHardwareDataType", "-detailLevel", "mini"]), + "memsize": text(["sysctl", "-n", "hw.memsize"]), "repo_head": text(["git", "-C", str(repo_root), "rev-parse", "HEAD"]), + "repo_status": text(["git", "-C", str(repo_root), "status", "--short"]), "ffmpeg": text(["ffmpeg", "-version"]).splitlines()[:1], + } + + +def strict_failures(result: dict[str, Any]) -> list[str]: + failures: list[str] = [] + telemetry = result.get("runtime_summary") or {} + lease = result.get("lease") or {} + dimensions = result.get("dimensions") or {} + hashes = result.get("hashes") or {} + phases = result.get("progress_phases") or [] + if telemetry.get("peak_rss_gib") is None: + failures.append("missing worker/process RSS peak") + if telemetry.get("peak_physical_footprint_gib") is None: + failures.append("missing authoritative worker physical footprint") + if result.get("runtime_policy", {}).get("auto_fast_video_engine") == "mlx": + if telemetry.get("peak_mlx_mib") is None: + failures.append("missing MLX allocator peak") + if not telemetry.get("mlx_runtime_identity"): + failures.append("missing profiled MLX runtime identity") + if telemetry.get("mlx_profile_status") not in {"success", "cancelled"}: + failures.append("missing terminal MLX profile status") + if not phases: + failures.append("missing generation phase samples") + if not result.get("cleanup_evidence"): + failures.append("missing explicit cleanup/post-cleanup telemetry") + if lease.get("running_probe", {}).get("observed") != "contended": + failures.append("shared Metal lease not held during local generation") + if lease.get("terminal_probe", {}).get("observed") != "acquired": + failures.append("shared Metal lease not released after cleanup") + for key in ("requested", "resolved", "actual"): + if not dimensions.get(key): + failures.append(f"missing {key} geometry") + resolved = dimensions.get("resolved") or {} + actual = dimensions.get("actual") or {} + for axis in ("width", "height"): + if resolved.get(axis) is None: + failures.append(f"missing resolved {axis}") + elif actual.get(axis) is None: + failures.append(f"missing actual {axis}") + elif resolved[axis] != actual[axis]: + failures.append( + f"resolved {axis} {resolved[axis]} does not match actual {axis} {actual[axis]}" + ) + for key in ("recipe_sha256", "prompt_sha256", "source_sha256", "repo_head"): + if not hashes.get(key): + failures.append(f"missing {key}") + return failures + + +def _peak(rows: list[dict[str, Any]], key: str) -> float | None: + values = [float(row[key]) for row in rows if row.get(key) is not None] + return max(values) if values else None + + +def run_case(case: MatrixCase, image_path: str | None, artifact_dir: Path, expect_mode: str, poll_seconds: float, cancel_during_run: bool = False) -> dict[str, Any]: + progress_before = http_json("GET", perf_config.STATUS_PATH) + if progress_before.get("status") in {"running"}: + raise RuntimeError(f"refusing to overlap existing generation: {progress_before}") + policy = http_json("GET", "/api/runtime-policy") + if expect_mode != "any" and policy.get("execution_mode") != expect_mode: + raise RuntimeError(f"runtime mode {policy.get('execution_mode')} does not match --expect-mode={expect_mode}") + initial_probe = probe_metal_lock() + if initial_probe["observed"] != "acquired": + raise RuntimeError(f"shared Metal lease is already contended: {initial_probe}") + + payload = case.payload(image_path) + submitted_at = time.time() + progress_samples: list[dict[str, Any]] = [] + runtime_samples: list[dict[str, Any]] = [] + process_samples: list[dict[str, Any]] = [] + running_probe: dict[str, Any] | None = None + cancel_sent = False + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(http_json, "POST", perf_config.GENERATE_PATH, payload, 1800) + while not future.done(): + progress = http_json("GET", perf_config.STATUS_PATH) + runtime = http_json("GET", "/api/runtime-telemetry") + progress_samples.append({"timestamp": time.time(), **progress}) + runtime_samples.append({"timestamp": time.time(), **runtime}) + if runtime.get("local_metal_lease_status") == "held": + if running_probe is None: + running_probe = probe_metal_lock() + owner = (running_probe or {}).get("holder_payload") or runtime.get("local_metal_lease_owner") or {} + if owner.get("pid"): + process_samples.append(process_sample(int(owner["pid"]))) + if cancel_during_run and not cancel_sent and progress.get("phase") == "inference": + http_json("POST", "/api/generate/cancel", {}) + cancel_sent = True + time.sleep(poll_seconds) + response = future.result() + + # The isolated MLX child publishes its terminal allocator peak and identity + # after process exit. Capture one post-job API sample so strict evidence does + # not depend on the polling interval racing the final flushed JSONL event. + terminal_runtime = http_json("GET", "/api/runtime-telemetry") + runtime_samples.append({"timestamp": time.time(), **terminal_runtime}) + terminal_probe = probe_metal_lock() + output_path = str((response or {}).get("video_path") or "") + actual = ffprobe(output_path) if output_path and Path(output_path).is_file() else {} + resolved_width = response.get("resolved_width") + resolved_height = response.get("resolved_height") + resolved_frames = ((case.duration * case.fps) // 8) * 8 + 1 + details = machine_details(Path(__file__).resolve().parents[2]) + phases = sorted({str(row.get("phase")) for row in progress_samples if row.get("phase")}) + cleanup = [row for row in runtime_samples if row.get("local_metal_lease_status") == "held" and row.get("active_pipeline") is None] + result: dict[str, Any] = { + "schema": "ltx.hd-benchmark.result.v1", "product": "ltx_desktop_electron", "case": asdict(case), + "response": response, "submitted_at": submitted_at, "finished_at": time.time(), "wall_seconds": time.time() - submitted_at, + "output_path": output_path or None, "runtime_policy": policy, "progress_phases": phases, "cleanup_evidence": cleanup, + "dimensions": { + "requested": {"resolution": case.resolution, "duration": case.duration, "fps": case.fps, "aspect_ratio": case.aspect_ratio}, + "resolved": {"width": resolved_width, "height": resolved_height, "frames": resolved_frames, "execution_mode": policy.get("execution_mode")}, + "actual": actual, + }, + "hashes": { + "recipe_sha256": canonical_sha(payload), "prompt_sha256": hashlib.sha256(PROMPT.encode()).hexdigest(), + "source_sha256": file_sha(image_path) if case.image_conditioned else canonical_sha({"source": "text_only"}), + "output_sha256": file_sha(output_path), "repo_head": details.get("repo_head"), "runtime_policy_sha256": canonical_sha(policy), + }, + "lease": {"initial_probe": initial_probe, "running_probe": running_probe or {}, "terminal_probe": terminal_probe}, + "runtime_summary": { + "samples": len(runtime_samples), "peak_rss_gib": max((_row.get("process_rss_mib", 0) for _row in runtime_samples), default=0) / 1024 if runtime_samples else None, + "peak_mlx_mib": max((_row.get("mlx_peak_mib") for _row in runtime_samples if _row.get("mlx_peak_mib") is not None), default=None), + "peak_mps_allocated_mib": max((_row.get("mps_allocated_mib") for _row in runtime_samples if _row.get("mps_allocated_mib") is not None), default=None), + "peak_mps_driver_mib": max((_row.get("mps_driver_mib") for _row in runtime_samples if _row.get("mps_driver_mib") is not None), default=None), + "peak_process_tree_rss_gib": _peak(process_samples, "rss_gib"), "peak_physical_footprint_gib": _peak(process_samples, "physical_footprint_gib"), + "peak_cpu_percent": _peak(process_samples, "cpu_percent"), "peak_gpu_utilization_percent": _peak(process_samples, "gpu_utilization_percent"), + "mlx_profile_status": terminal_runtime.get("mlx_profile_status"), + "mlx_profile_phase": terminal_runtime.get("mlx_profile_phase"), + "mlx_profile_path": terminal_runtime.get("mlx_profile_path"), + "mlx_profile_sampled_at": terminal_runtime.get("mlx_profile_sampled_at"), + "mlx_runtime_identity": terminal_runtime.get("mlx_runtime_identity"), + }, + "machine": details, "cancel_sent": cancel_sent, + } + result["strict_failures"] = strict_failures(result) + result["strict_status"] = "PASS" if not result["strict_failures"] and response.get("status") == "complete" else "FAIL" + case_dir = artifact_dir / case.case_id + case_dir.mkdir(parents=True, exist_ok=True) + (case_dir / "request.json").write_text(json.dumps(payload, indent=2), encoding="utf-8") + (case_dir / "progress_samples.jsonl").write_text("".join(json.dumps(row, separators=(",", ":")) + "\n" for row in progress_samples), encoding="utf-8") + (case_dir / "runtime_samples.jsonl").write_text("".join(json.dumps(row, separators=(",", ":")) + "\n" for row in runtime_samples), encoding="utf-8") + (case_dir / "process_samples.jsonl").write_text("".join(json.dumps(row, separators=(",", ":")) + "\n" for row in process_samples), encoding="utf-8") + (case_dir / "result.json").write_text(json.dumps(result, indent=2, default=str), encoding="utf-8") + return result + + +def add_reference_quality(results: list[dict[str, Any]], reference_manifest: Path | None, artifact_dir: Path) -> None: + if reference_manifest is None: + return + prior = json.loads(reference_manifest.read_text(encoding="utf-8")) + prior_rows = prior if isinstance(prior, list) else prior.get("results", []) + by_case = {row.get("case", {}).get("case_id"): row for row in prior_rows} + blinded, answer = [], [] + for row in results: + case_id = row["case"]["case_id"] + ref = by_case.get(case_id) or {} + reference, candidate = ref.get("output_path"), row.get("output_path") + if not reference or not candidate or not Path(reference).is_file() or not Path(candidate).is_file(): + continue + row["quality_vs_reference"] = quality_metrics(reference, candidate) + if canonical_sha({"case": case_id, "candidate": candidate})[-1] in "02468ace": + pair = [("reference", reference), ("candidate", candidate)] + else: + pair = [("candidate", candidate), ("reference", reference)] + blinded.append({"pair_id": case_id, "A": pair[0][1], "B": pair[1][1], "questions": ["Which has better motion coherence?", "Which preserves detail?", "Which audio is cleaner?", "Any seams or freezes?"]}) + answer.append({"pair_id": case_id, "A": pair[0][0], "B": pair[1][0]}) + (artifact_dir / "blinded_review.json").write_text(json.dumps(blinded, indent=2), encoding="utf-8") + (artifact_dir / "blinded_review_key.json").write_text(json.dumps(answer, indent=2), encoding="utf-8") + + +def verify_kernel_crash_release(artifact_dir: Path) -> dict[str, Any]: + """Prove kernel-owned flock release after an owner is SIGKILLed, without crashing either app.""" + helper = "import fcntl,os,sys,time; f=open(sys.argv[1],'a+'); fcntl.flock(f.fileno(),fcntl.LOCK_EX); print(os.getpid(),flush=True); time.sleep(60)" + process = subprocess.Popen([sys.executable, "-c", helper, str(LOCK_PATH)], stdout=subprocess.PIPE, text=True) + assert process.stdout is not None + process.stdout.readline() + while_alive = probe_metal_lock() + process.kill() + process.wait(timeout=10) + after_sigkill = probe_metal_lock() + result = { + "scope": "kernel flock owner SIGKILL simulation; neither product server was crashed", + "while_owner_alive": while_alive, + "after_sigkill": after_sigkill, + "pass": while_alive["observed"] == "contended" and after_sigkill["observed"] == "acquired", + } + artifact_dir.mkdir(parents=True, exist_ok=True) + (artifact_dir / "crash_release.json").write_text(json.dumps(result, indent=2), encoding="utf-8") + return result + + +def write_summary(results: list[dict[str, Any]], artifact_dir: Path, command: str) -> None: + lines = ["# LTX Desktop Electron HD benchmark", "", f"Command: `{command}`", "", "| case | mode | status | wall s | footprint GiB | MLX MiB | SHA-256 | strict gaps |", "|---|---|---:|---:|---:|---:|---|---|"] + for row in results: + summary = row.get("runtime_summary") or {} + lines.append(f"| {row['case']['case_id']} | {row.get('runtime_policy', {}).get('execution_mode')} | {row.get('strict_status')} | {row.get('wall_seconds', 0):.2f} | {summary.get('peak_physical_footprint_gib')} | {summary.get('peak_mlx_mib')} | {(row.get('hashes') or {}).get('output_sha256') or '—'} | {'; '.join(row.get('strict_failures') or []) or '—'} |") + (artifact_dir / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8") + (artifact_dir / "results.json").write_text(json.dumps({"schema": "ltx.hd-benchmark.manifest.v1", "results": results}, indent=2, default=str), encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--execute", action="store_true") + parser.add_argument("--case", action="append", default=[]) + parser.add_argument("--runs-per-case", type=int, default=1, choices=range(1, 11)) + parser.add_argument("--expect-mode", choices=("any", "eager", "low_ram"), default="any") + parser.add_argument("--image", type=Path, default=Path(__file__).resolve().parent / "test_assets/reference_image.png") + parser.add_argument("--artifacts", type=Path, default=perf_config.RUNS_DIR / f"hd_matrix_{datetime.now().strftime('%Y%m%d_%H%M%S')}") + parser.add_argument("--reference-manifest", type=Path) + parser.add_argument("--poll-seconds", type=float, default=1.0) + parser.add_argument("--verify-cancel-release", action="store_true", help="cancel only the harness's own first selected run during inference") + parser.add_argument("--verify-crash-release", action="store_true", help="no-model SIGKILL helper proving kernel flock release") + args = parser.parse_args() + matrix = default_matrix() + if args.case: + wanted = set(args.case) + matrix = [case for case in matrix if case.case_id in wanted] + missing = wanted - {case.case_id for case in matrix} + if missing: + parser.error(f"unknown cases: {sorted(missing)}") + plan = { + "schema": "ltx.hd-benchmark.plan.v1", "product": "ltx_desktop_electron", + "production_entrypoint": f"POST {perf_config.GENERATE_PATH}", "production_lease": str(LOCK_PATH), + "expect_mode": args.expect_mode, "runs_per_case": args.runs_per_case, "cases": [asdict(case) for case in matrix], + "notes": [ + "Run once with --expect-mode=eager and once with --expect-mode=low_ram; runtime policy, not a request-only toggle, owns the app mode.", + "Distilled Fast T2V/I2V cannot use TeaCache and this harness never presents it as a switch.", + "Electron Fast exposes only its production automatic policy; explicit modality-tiling candidates are out of scope for this matrix.", + "The production flock serializes local heavy work; cloud/API/CPU-only work remains outside it.", + ], + } + if args.verify_crash_release and not args.execute: + args.artifacts.mkdir(parents=True, exist_ok=True) + result = verify_kernel_crash_release(args.artifacts) + print(json.dumps(result, indent=2)) + return 0 if result["pass"] else 1 + if not args.execute: + print(json.dumps(plan, indent=2)) + return 0 + if any(case.image_conditioned for case in matrix) and not args.image.is_file(): + parser.error(f"image fixture missing: {args.image}") + perf_config.wait_for_backend() + args.artifacts.mkdir(parents=True, exist_ok=True) + (args.artifacts / "plan.json").write_text(json.dumps(plan, indent=2), encoding="utf-8") + results: list[dict[str, Any]] = [] + cancel_pending = args.verify_cancel_release + for case in matrix: + for run_index in range(args.runs_per_case): + run_case_value = case if args.runs_per_case == 1 else MatrixCase(f"{case.case_id}_run{run_index + 1}", case.resolution, case.duration, case.fps, case.aspect_ratio, case.image_conditioned) + print(f"[qa] {run_case_value.case_id}", flush=True) + results.append(run_case(run_case_value, str(args.image.resolve()), args.artifacts, args.expect_mode, args.poll_seconds, cancel_during_run=cancel_pending)) + cancel_pending = False + add_reference_quality(results, args.reference_manifest, args.artifacts) + if args.verify_crash_release: + verify_kernel_crash_release(args.artifacts) + write_summary(results, args.artifacts, " ".join(sys.argv)) + print(args.artifacts) + return 1 if any(row.get("strict_status") != "PASS" for row in results) else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/runtime_config/mlx_runtime.py b/backend/runtime_config/mlx_runtime.py new file mode 100644 index 000000000..4eab92e18 --- /dev/null +++ b/backend/runtime_config/mlx_runtime.py @@ -0,0 +1,149 @@ +"""Portable discovery and exact identity validation for the external MLX runtime.""" + +from __future__ import annotations + +import json +import os +import shlex +import shutil +import subprocess +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path + +from runtime_config.runtime_policy import MLX_RUNTIME_REVISION, MLX_RUNTIME_VERSION + +@dataclass(frozen=True) +class MLXRuntimeDiscovery: + command_prefix: tuple[str, ...] | None + source: str + version: str + revision: str | None + dirty: bool | None + core_version: str | None + mlx_version: str | None + compatible: bool + reason: str + + +def _identity_python_for_entrypoint(executable: str) -> str | None: + """Resolve the interpreter from an ordinary venv console-script shebang.""" + try: + first_line = Path(executable).read_bytes().splitlines()[0].decode("utf-8") + except (OSError, UnicodeDecodeError, IndexError): + return None + if not first_line.startswith("#!"): + return None + parts = shlex.split(first_line[2:]) + if not parts: + return None + if Path(parts[0]).name == "env" and len(parts) > 1: + return shutil.which(parts[1]) + return parts[0] + + +def _candidate_runtimes() -> list[tuple[tuple[str, ...], str | None, str]]: + candidates: list[tuple[tuple[str, ...], str | None, str]] = [] + configured_python = os.environ.get("LTX_MLX_PYTHON") + if configured_python: + python = str(Path(configured_python).expanduser()) + candidates.append(((python, "-m", "ltx_pipelines_mlx.cli"), python, "LTX_MLX_PYTHON")) + + default_python = Path("~/video-models/ltx-2-mlx/.venv/bin/python").expanduser() + candidates.append( + ( + (str(default_python), "-m", "ltx_pipelines_mlx.cli"), + str(default_python), + str(default_python), + ) + ) + + configured_executable = os.environ.get("LTX_MLX_EXECUTABLE") + executable = shutil.which(configured_executable or "ltx-2-mlx") + if executable is not None: + candidates.append(((executable,), _identity_python_for_entrypoint(executable), executable)) + + deduplicated: list[tuple[tuple[str, ...], str | None, str]] = [] + seen: set[tuple[str, ...]] = set() + for candidate in candidates: + if candidate[0] not in seen: + seen.add(candidate[0]) + deduplicated.append(candidate) + return deduplicated + + +def _probe_candidate( + command_prefix: tuple[str, ...], + identity_python: str | None, + source: str, +) -> MLXRuntimeDiscovery | None: + if identity_python is None or not Path(identity_python).expanduser().is_file(): + return None + try: + result = subprocess.run( + [identity_python, "-m", "ltx_pipelines_mlx.utils.runtime_info"], + capture_output=True, + check=False, + text=True, + timeout=10, + ) + lines = result.stdout.strip().splitlines() + if result.returncode != 0 or not lines: + return None + identity = json.loads(lines[-1]) + except (OSError, subprocess.SubprocessError, json.JSONDecodeError): + return None + + version = os.environ.get("LTX_MLX_RUNTIME_VERSION") or identity.get("runtime_version") + revision = os.environ.get("LTX_MLX_RUNTIME_REVISION") or identity.get("runtime_commit") + dirty = identity.get("runtime_dirty") + exact_version = version == MLX_RUNTIME_VERSION + exact_revision = revision == MLX_RUNTIME_REVISION + clean = dirty is not True + compatible = exact_version and exact_revision and clean + mismatches: list[str] = [] + if not exact_version: + mismatches.append(f"version {version or 'unknown'} != {MLX_RUNTIME_VERSION}") + if not exact_revision: + mismatches.append(f"revision {revision or 'unknown'} != {MLX_RUNTIME_REVISION}") + if not clean: + mismatches.append("runtime checkout is dirty") + reason = "Exact pinned MLX runtime identity verified." if compatible else "; ".join(mismatches) + return MLXRuntimeDiscovery( + command_prefix=command_prefix if compatible else None, + source=source, + version=version or "unknown", + revision=revision, + dirty=dirty, + core_version=identity.get("core_version"), + mlx_version=identity.get("mlx_version"), + compatible=compatible, + reason=reason, + ) + + +@lru_cache(maxsize=1) +def discover_mlx_runtime() -> MLXRuntimeDiscovery: + """Prefer an explicit Python, then the sibling runtime, then a PATH entrypoint.""" + first_detected: MLXRuntimeDiscovery | None = None + for command_prefix, identity_python, source in _candidate_runtimes(): + discovered = _probe_candidate(command_prefix, identity_python, source) + if discovered is None: + continue + if discovered.compatible: + return discovered + if first_detected is None: + first_detected = discovered + if first_detected is not None: + return first_detected + return MLXRuntimeDiscovery( + command_prefix=None, + source="not found", + version="not installed", + revision=None, + dirty=None, + core_version=None, + mlx_version=None, + compatible=False, + reason="No discoverable MLX Python runtime or ltx-2-mlx entrypoint was found.", + ) diff --git a/backend/runtime_config/runtime_config.py b/backend/runtime_config/runtime_config.py index fb098549d..e45ec1778 100644 --- a/backend/runtime_config/runtime_config.py +++ b/backend/runtime_config/runtime_config.py @@ -4,10 +4,17 @@ from dataclasses import dataclass from pathlib import Path +from typing import Literal import torch -from runtime_config.runtime_policy import LocalGenerationMode +from runtime_config.runtime_policy import ( + FastVideoEngineDecision, + FastVideoEnginePreference, + LocalGenerationMode, + MLX_BF16_MODEL_SOURCE, + decide_fast_video_engine, +) @dataclass @@ -24,6 +31,18 @@ class RuntimeConfig: default_negative_prompt: str dev_mode: bool backend_port: int + fast_video_engine_preference: FastVideoEnginePreference = "auto" + mlx_runtime_eligible: bool = False + mlx_model_cached: bool = False + mlx_model_source: str = MLX_BF16_MODEL_SOURCE + mlx_model_variant: Literal["bf16", "q8"] = "bf16" + mlx_runtime_version: str = "not installed" + mlx_runtime_revision: str | None = None + mlx_runtime_source: str = "not found" + mlx_runtime_dirty: bool | None = None + mlx_core_version: str | None = None + mlx_framework_version: str | None = None + available_ram_gb: int | None = None hf_oauth_client_id: str = "" lora_catalog_source: str = "" # Bundled catalog used as a fallback when lora_catalog_source is a URL that fails to fetch. @@ -33,3 +52,12 @@ class RuntimeConfig: def force_api_generations(self) -> bool: """Derived: local generation is unavailable for this runtime.""" return self.local_generations_mode == "unsupported" + + def decide_fast_video_engine(self, *, use_local_text_encoding: bool) -> FastVideoEngineDecision: + return decide_fast_video_engine( + preference=self.fast_video_engine_preference, + mlx_runtime_eligible=self.mlx_runtime_eligible, + mlx_model_cached=self.mlx_model_cached, + use_local_text_encoding=use_local_text_encoding, + mlx_quality_qualified=self.mlx_model_variant == "bf16", + ) diff --git a/backend/runtime_config/runtime_policy.py b/backend/runtime_config/runtime_policy.py index 91bbcce41..a6ba9b73e 100644 --- a/backend/runtime_config/runtime_policy.py +++ b/backend/runtime_config/runtime_policy.py @@ -2,6 +2,7 @@ from __future__ import annotations +from dataclasses import dataclass from typing import Literal LocalGenerationMode = Literal[ @@ -9,6 +10,26 @@ "streaming_models_loading", "unsupported", ] +FastVideoEngine = Literal["torch", "mlx"] +FastVideoEnginePreference = Literal["auto", "torch", "mlx"] +FastVideoExecutionMode = Literal["eager", "low_ram", "unsupported"] + +MLX_RUNTIME_VERSION = "0.14.20.dev1" +MLX_RUNTIME_REVISION = "3171bac4ba901c0237faea2678c34034b37abc2a" +MLX_BF16_MODEL_SOURCE = "dgrauet/ltx-2.3-mlx" +MLX_Q8_MODEL_SOURCE = "dgrauet/ltx-2.3-mlx-q8" +TORCH_RUNTIME_REVISION = "9377758131b1ffde4b7f766804590a6617bf2ab9" + +# BF16 block streaming was validated by the MLX runtime for 32 GB Macs. Eager +# materialization needs substantially more headroom, so keep the automatic +# threshold conservative and use block streaming below it. +MLX_BF16_EAGER_FLOOR_GB = 64 + + +@dataclass(frozen=True, slots=True) +class FastVideoEngineDecision: + engine: FastVideoEngine + reason: str # Below this, local generation isn't viable at all on Darwin. Roughly matches the # measured ~13 GB RSS of the distilled pipeline streaming on an M4 Pro, plus margin @@ -116,3 +137,80 @@ def streaming_prefetch_count_for_mode(mode: LocalGenerationMode) -> int | None: if mode == "streaming_models_loading": return 2 raise AssertionError(f"Unexpected LocalGenerationMode: {mode!r}") + + +def decide_fast_video_execution_mode( + engine: FastVideoEngine, + local_mode: LocalGenerationMode, + available_ram_gb: int | None, +) -> FastVideoExecutionMode: + """Choose eager vs low-RAM loading for the selected Fast pipeline. + + Torch keeps its existing, separately qualified policy. MLX BF16 can stream + transformer blocks directly from mmap and therefore uses a lower, explicit + eager threshold without changing the policy used by Retake/Extend/A2V/ + IC-LoRA Torch fallbacks. + """ + if local_mode == "unsupported": + return "unsupported" + if engine == "torch": + return "eager" if local_mode == "full_models_loading" else "low_ram" + if available_ram_gb is not None and available_ram_gb >= MLX_BF16_EAGER_FLOOR_GB: + return "eager" + return "low_ram" + + +def decide_fast_video_engine( + *, + preference: FastVideoEnginePreference, + mlx_runtime_eligible: bool, + mlx_model_cached: bool, + use_local_text_encoding: bool, + mlx_quality_qualified: bool = True, +) -> FastVideoEngineDecision: + """Resolve Fast T2V/I2V without silently dropping prepared embeddings. + + ``auto`` is capability-aware: MLX is selected only for requests whose text + is encoded locally by the MLX pipeline, and only when both runtime and BF16 + weights are already available. An explicit MLX preference may download the + configured model on first use, but still fails closed to Torch when the + platform/runtime cannot execute MLX. + """ + if preference == "torch": + return FastVideoEngineDecision("torch", "Torch was explicitly selected.") + if preference == "mlx": + if mlx_runtime_eligible: + suffix = ( + " BF16 weights are cached." + if mlx_model_cached + else " BF16 weights are not cached and may download on first use." + ) + return FastVideoEngineDecision("mlx", "MLX was explicitly selected." + suffix) + return FastVideoEngineDecision( + "torch", + "MLX was explicitly requested but is unavailable on this runtime; using Torch.", + ) + if not use_local_text_encoding: + return FastVideoEngineDecision( + "torch", + "Prepared/API text embeddings require the feature-complete Torch pipeline.", + ) + if not mlx_quality_qualified: + return FastVideoEngineDecision( + "torch", + "MLX q8 is expert-only and is never auto-selected after quality qualification.", + ) + if not mlx_runtime_eligible: + return FastVideoEngineDecision( + "torch", + "MLX auto-selection requires Apple Silicon, MPS, and the pinned MLX runtime.", + ) + if not mlx_model_cached: + return FastVideoEngineDecision( + "torch", + "MLX BF16 weights are not cached; auto mode avoids an unannounced model download.", + ) + return FastVideoEngineDecision( + "mlx", + "MLX auto-selected for Fast T2V/I2V with local text encoding and cached BF16 weights.", + ) diff --git a/backend/services/fast_video_pipeline/mlx_fast_video_pipeline.py b/backend/services/fast_video_pipeline/mlx_fast_video_pipeline.py new file mode 100644 index 000000000..a1745cb99 --- /dev/null +++ b/backend/services/fast_video_pipeline/mlx_fast_video_pipeline.py @@ -0,0 +1,231 @@ +"""Apple Silicon MLX adapter for parity-proven Fast T2V/I2V requests.""" + +from __future__ import annotations + +import logging +import os +import signal +import subprocess +import threading +from typing import Final + +from api_types import ImageConditioningInput +from runtime_config.mlx_runtime import discover_mlx_runtime +from runtime_config.runtime_policy import MLX_BF16_MODEL_SOURCE +from services.fast_video_pipeline.mlx_profile import ( + allocate_mlx_profile_path, + begin_mlx_profile, + finish_mlx_profile, +) + +logger = logging.getLogger(__name__) +_MLX_DISTILLED_SPATIAL_GRID = 64 +_sidecar_lock = threading.Lock() +_active_sidecar_process: subprocess.Popen[bytes] | None = None +_cancelled_sidecar_pids: set[int] = set() + + +def get_active_mlx_sidecar_pid() -> int | None: + with _sidecar_lock: + process = _active_sidecar_process + return process.pid if process is not None and process.poll() is None else None + + +def _register_active_sidecar(process: subprocess.Popen[bytes]) -> None: + global _active_sidecar_process + with _sidecar_lock: + if _active_sidecar_process is not None and _active_sidecar_process.poll() is None: + raise RuntimeError("An MLX sidecar is already active") + _active_sidecar_process = process + + +def _release_active_sidecar(process: subprocess.Popen[bytes]) -> bool: + global _active_sidecar_process + with _sidecar_lock: + was_cancelled = process.pid in _cancelled_sidecar_pids + _cancelled_sidecar_pids.discard(process.pid) + if _active_sidecar_process is process: + _active_sidecar_process = None + return was_cancelled + + +def cancel_active_mlx_sidecar() -> bool: + """Kill the entire per-job MLX process group so cancellation returns memory.""" + with _sidecar_lock: + process = _active_sidecar_process + if process is None or process.poll() is not None: + return False + _cancelled_sidecar_pids.add(process.pid) + pid = process.pid + + try: + # start_new_session=True makes the sidecar PID its process-group ID. A + # hard group kill is intentional: no MLX worker/grandchild may retain + # unified memory after the user cancels the job. + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + pass + logger.info("Killed MLX sidecar process group pid=%d after cancellation", pid) + return True + + +class MLXFastVideoPipeline: + """Wrap ``ltx-pipelines-mlx`` without importing MLX on non-Darwin hosts. + + This adapter intentionally covers only the distilled Fast T2V/I2V surface. + A2V, Retake, Extend, IC-LoRA, prepared/API embeddings, and the rest of the + product continue to use the official Torch pipeline. + """ + + pipeline_kind: Final = "fast" + + @staticmethod + def create( + checkpoint_path: str, + gemma_root: str | None, + upsampler_path: str, + device: object, + streaming_prefetch_count: int | None, + loras: list[tuple[str, float]] | None = None, + ) -> "MLXFastVideoPipeline": + del checkpoint_path, gemma_root, upsampler_path, device + return MLXFastVideoPipeline( + model_source=os.environ.get("LTX_MLX_MODEL_ID", MLX_BF16_MODEL_SOURCE), + low_ram=streaming_prefetch_count is not None, + loras=loras or [], + ) + + def __init__( + self, + *, + model_source: str, + low_ram: bool, + loras: list[tuple[str, float]], + ) -> None: + self._model_source = model_source + self._low_ram = low_ram + self._loras = loras + model_precision = os.environ.get("LTX_MLX_MODEL_VARIANT", "bf16").strip().lower() + self._model_precision = model_precision if model_precision in {"bf16", "q8", "q4"} else "unknown" + runtime = discover_mlx_runtime() + command_prefix = runtime.command_prefix + if command_prefix is None: + raise RuntimeError( + "Compatible MLX sidecar unavailable: " + f"{runtime.reason}. Set LTX_MLX_PYTHON to the pinned runtime or select Torch." + ) + self._command_prefix: tuple[str, ...] = command_prefix + # The MLX VAE decoder estimates tile size from this budget. Keep the + # setting child-local so the lean FastAPI process never acquires MLX + # allocator state or job-specific runtime configuration. + self._vae_decode_budget_gb = "2" if low_ram else "8" + logger.info( + "Created MLX Fast sidecar pipeline command=%s model=%s mode=%s loras=%d", + " ".join(self._command_prefix), + model_source, + "low_ram" if low_ram else "eager", + len(loras), + ) + + def generate( + self, + prompt: str, + seed: int, + height: int, + width: int, + num_frames: int, + frame_rate: float, + images: list[ImageConditioningInput], + output_path: str, + ) -> None: + if height % _MLX_DISTILLED_SPATIAL_GRID or width % _MLX_DISTILLED_SPATIAL_GRID: + raise ValueError( + "MLX distilled output dimensions must both be divisible by " + f"{_MLX_DISTILLED_SPATIAL_GRID}; got {width}x{height}" + ) + profile_path = allocate_mlx_profile_path(output_path) + command: list[str] = [ + *self._command_prefix, + "generate", + "--distilled", + "--prompt", + prompt, + "--output", + output_path, + "--model", + self._model_source, + "--model-precision", + self._model_precision, + "--height", + str(height), + "--width", + str(width), + "--frames", + str(num_frames), + "--frame-rate", + str(frame_rate), + "--seed", + str(seed), + "--profile-json", + str(profile_path), + ] + if self._low_ram: + command.extend(["--low-ram", "--auto-tiling"]) + for image in images: + command.extend( + [ + "--image", + image.path, + str(image.frame_idx), + str(image.strength), + ] + ) + for path, strength in self._loras: + command.extend(["--lora", path, str(strength)]) + child_env = os.environ.copy() + child_env["LTX2_VAE_DECODE_BUDGET_GB"] = self._vae_decode_budget_gb + # One fresh process group per generation is a memory invariant. A + # persistent MLX worker retained ~50.8 GiB and made request 2 slower; + # process exit is the allocator teardown boundary we can prove. + process = subprocess.Popen(command, env=child_env, start_new_session=True) + _register_active_sidecar(process) + begin_mlx_profile(profile_path) + return_code: int | None = None + was_cancelled = False + try: + return_code = process.wait() + finally: + was_cancelled = _release_active_sidecar(process) + profile_status = ( + "cancelled" + if was_cancelled + else "success" + if return_code == 0 + else "error" + ) + finish_mlx_profile(profile_path, profile_status) + if was_cancelled: + raise RuntimeError("Generation was cancelled") + if return_code != 0: + raise subprocess.CalledProcessError(return_code or -1, command) + + def warmup(self, output_path: str) -> None: + self.generate( + prompt="test warmup", + seed=42, + height=256, + width=384, + num_frames=9, + frame_rate=8, + images=[], + output_path=output_path, + ) + try: + os.unlink(output_path) + except FileNotFoundError: + pass + + def compile_transformer(self) -> None: + # MLX compiles/evaluates its own lazy graphs; torch.compile is not + # applicable. Keeping the protocol method makes selection transparent. + logger.info("Skipping torch.compile for the MLX Fast pipeline") diff --git a/backend/services/fast_video_pipeline/mlx_profile.py b/backend/services/fast_video_pipeline/mlx_profile.py new file mode 100644 index 000000000..7fe8e6d3b --- /dev/null +++ b/backend/services/fast_video_pipeline/mlx_profile.py @@ -0,0 +1,201 @@ +"""Crash-resilient ingestion for per-job MLX JSONL profiles.""" + +from __future__ import annotations + +import json +import math +import os +import threading +import uuid +from dataclasses import dataclass, replace +from datetime import UTC, datetime +from pathlib import Path +from typing import Literal, cast + +MLXProfileStatus = Literal["running", "success", "error", "cancelled"] +_GIB_TO_MIB = 1024 +_PROFILE_IDENTITY_KEYS = { + "runtime_commit", + "runtime_dirty", + "runtime_version", + "core_version", + "mlx_version", + "mlx_metal_version", + "device_name", + "device_architecture", + "device_memory_bytes", + "device_recommended_working_set_bytes", + "runtime_family", + "device_family", +} + + +@dataclass(frozen=True) +class MLXProfileSnapshot: + profile_path: str + status: MLXProfileStatus + sampled_at: str | None = None + phase: str | None = None + active_mib: int | None = None + cache_mib: int | None = None + peak_mib: int | None = None + runtime_identity: dict[str, object] | None = None + + +_profile_lock = threading.Lock() +_active_profile_path: Path | None = None +_last_profile_snapshot: MLXProfileSnapshot | None = None + + +def allocate_mlx_profile_path(output_path: str) -> Path: + """Allocate a durable, prompt-free profile path beside generated outputs.""" + output = Path(output_path).expanduser() + profile_dir = output.parent / ".mlx-profiles" + profile_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + try: + os.chmod(profile_dir, 0o700) + except OSError: + pass + return profile_dir / f"{output.stem}-{uuid.uuid4().hex}.jsonl" + + +def _finite_number(value: object) -> float | None: + if not isinstance(value, (float, int)) or isinstance(value, bool): + return None + number = float(value) + return number if math.isfinite(number) else None + + +def _mib(value: object) -> int | None: + number = _finite_number(value) + return round(number * _GIB_TO_MIB) if number is not None else None + + +def _sampled_at(value: object) -> str | None: + number = _finite_number(value) + if number is None: + return None + try: + return datetime.fromtimestamp(number, tz=UTC).isoformat() + except (OverflowError, OSError, ValueError): + return None + + +def read_mlx_profile(path: Path) -> MLXProfileSnapshot | None: + """Read all complete JSONL records; tolerate a concurrently-written tail.""" + try: + lines = path.read_text(encoding="utf-8").splitlines() + except OSError: + return None + + status: MLXProfileStatus = "running" + sampled_at = None + phase = None + active_mib = None + cache_mib = None + peak_mib = None + runtime_identity: dict[str, object] | None = None + records = 0 + for line in lines: + try: + raw_record: object = json.loads(line) + except (json.JSONDecodeError, TypeError): + continue + if not isinstance(raw_record, dict): + continue + record = cast(dict[str, object], raw_record) + records += 1 + sampled_at = _sampled_at(record.get("timestamp_unix_seconds")) or sampled_at + event = record.get("event") + if event == "run_start": + status = "running" + metadata_value = record.get("metadata") + if isinstance(metadata_value, dict): + metadata = cast(dict[str, object], metadata_value) + runtime_identity = { + key: value + for key, value in metadata.items() + if key in _PROFILE_IDENTITY_KEYS + and isinstance(value, (str, int, float, bool)) + } + elif event == "run_end": + status = "success" + elif event == "run_error": + status = "error" + phase_value = record.get("phase") + if isinstance(phase_value, str): + phase = phase_value + + active_candidate = _mib(record.get("mlx_active_gb")) + cache_candidate = _mib(record.get("mlx_cache_gb")) + if active_candidate is not None: + active_mib = active_candidate + if cache_candidate is not None: + cache_mib = cache_candidate + candidates = ( + _mib(record.get("mlx_peak_gb")), + _mib(record.get("observed_peak_mlx_gb")), + ) + for candidate in candidates: + if candidate is not None: + peak_mib = max(peak_mib or 0, candidate) + + if records == 0: + return None + return MLXProfileSnapshot( + profile_path=str(path), + status=status, + sampled_at=sampled_at, + phase=phase, + active_mib=active_mib, + cache_mib=cache_mib, + peak_mib=peak_mib, + runtime_identity=runtime_identity, + ) + + +def begin_mlx_profile(path: Path) -> None: + global _active_profile_path, _last_profile_snapshot + with _profile_lock: + _active_profile_path = path + _last_profile_snapshot = MLXProfileSnapshot( + profile_path=str(path), + status="running", + ) + + +def finish_mlx_profile(path: Path, status: MLXProfileStatus) -> None: + global _active_profile_path, _last_profile_snapshot + snapshot = read_mlx_profile(path) + if snapshot is None: + snapshot = MLXProfileSnapshot(profile_path=str(path), status=status) + else: + snapshot = replace(snapshot, status=status) + with _profile_lock: + if _active_profile_path == path: + _active_profile_path = None + _last_profile_snapshot = snapshot + + +def get_mlx_profile_snapshot() -> MLXProfileSnapshot | None: + """Return the live flushed profile, or the last terminal job snapshot.""" + global _last_profile_snapshot + with _profile_lock: + active_path = _active_profile_path + fallback = _last_profile_snapshot + if active_path is None: + return fallback + live = read_mlx_profile(active_path) + if live is None: + return fallback + with _profile_lock: + if _active_profile_path == active_path: + _last_profile_snapshot = live + return live + + +def reset_mlx_profile_for_tests() -> None: + global _active_profile_path, _last_profile_snapshot + with _profile_lock: + _active_profile_path = None + _last_profile_snapshot = None diff --git a/backend/services/local_metal_lease.py b/backend/services/local_metal_lease.py new file mode 100644 index 000000000..0bf1e1b7d --- /dev/null +++ b/backend/services/local_metal_lease.py @@ -0,0 +1,187 @@ +"""Cross-product advisory lease for heavy local Apple Silicon inference.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from datetime import UTC, datetime +import json +import logging +import os +from pathlib import Path +import socket +import sys +import threading +import time +from typing import ContextManager, Literal, TypedDict, cast + +logger = logging.getLogger(__name__) + +LOCAL_METAL_LOCK_PATH = ( + Path.home() / "Library" / "Application Support" / "LTX Shared" / "local-metal.lock" +) +_POLL_SECONDS = 0.25 + + +class LocalMetalLeaseSnapshot(TypedDict): + status: Literal["idle", "waiting", "held"] + reason: str | None + waited_seconds: float + owner: dict[str, object] | None + + +_snapshot_lock = threading.Lock() +_snapshot: LocalMetalLeaseSnapshot = { + "status": "idle", + "reason": None, + "waited_seconds": 0.0, + "owner": None, +} + + +def get_local_metal_lease_snapshot() -> LocalMetalLeaseSnapshot: + with _snapshot_lock: + return { + "status": _snapshot["status"], + "reason": _snapshot["reason"], + "waited_seconds": _snapshot["waited_seconds"], + "owner": dict(_snapshot["owner"]) if _snapshot["owner"] is not None else None, + } + + +def _set_snapshot( + status: Literal["idle", "waiting", "held"], + *, + reason: str | None = None, + waited_seconds: float = 0.0, + owner: dict[str, object] | None = None, +) -> None: + with _snapshot_lock: + _snapshot.update( + status=status, + reason=reason, + waited_seconds=waited_seconds, + owner=owner, + ) + + +def _read_owner(fd: int) -> dict[str, object] | None: + try: + os.lseek(fd, 0, os.SEEK_SET) + payload = os.read(fd, 16 * 1024).decode("utf-8") + parsed = json.loads(payload) + return cast(dict[str, object], parsed) if isinstance(parsed, dict) else None + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + return None + + +def _emit(event: str, **fields: object) -> None: + logger.info("%s", json.dumps({"event": event, **fields}, sort_keys=True)) + + +@contextmanager +def local_metal_lease( + *, + job_id: str, + workload: str, + reason: str, + is_cancelled: Callable[[], bool], + on_wait: Callable[[float, dict[str, object] | None], None] | None = None, +) -> Iterator[None]: + """Acquire the shared Metal lock, polling so cancellation remains responsive.""" + if sys.platform != "darwin": + yield + return + + import fcntl + + path = LOCAL_METAL_LOCK_PATH + path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(path, os.O_RDWR | os.O_CREAT, 0o600) + os.chmod(path, 0o600) + started = time.monotonic() + acquired = False + metadata: dict[str, object] = { + "schema": "ltx.local-metal-lock.v1", + "product": "LTX Desktop", + "pid": os.getpid(), + "ppid": os.getppid(), + "job_id": job_id, + "workload": workload, + "reason": reason, + "acquired_at_utc": datetime.now(UTC).isoformat(), + "host": socket.gethostname(), + } + try: + while True: + if is_cancelled(): + raise RuntimeError("Generation cancelled while waiting for the local Metal accelerator") + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + acquired = True + break + except BlockingIOError: + waited = time.monotonic() - started + owner = _read_owner(fd) + _set_snapshot( + "waiting", + reason=reason, + waited_seconds=waited, + owner=owner, + ) + _emit( + "local_metal_lock_wait", + path=str(path), + product="LTX Desktop", + pid=os.getpid(), + job_id=job_id, + reason=reason, + waited_seconds=round(waited, 3), + owner=owner, + ) + if on_wait is not None: + on_wait(waited, owner) + time.sleep(_POLL_SECONDS) + + waited = time.monotonic() - started + encoded = json.dumps(metadata, sort_keys=True).encode("utf-8") + os.ftruncate(fd, 0) + os.lseek(fd, 0, os.SEEK_SET) + os.write(fd, encoded) + os.fsync(fd) + _set_snapshot("held", reason=reason, waited_seconds=waited, owner=metadata) + _emit( + "local_metal_lock_acquired", + path=str(path), + **metadata, + waited_seconds=round(waited, 3), + ) + yield + finally: + if acquired: + fcntl.flock(fd, fcntl.LOCK_UN) + _emit( + "local_metal_lock_released", + path=str(path), + product="LTX Desktop", + pid=os.getpid(), + job_id=job_id, + reason=reason, + ) + os.close(fd) + _set_snapshot("idle") + + +class LocalMetalLeaseHandle: + """Explicit-lifetime wrapper for handlers whose existing try/finally owns cleanup.""" + + def __init__(self, lease: ContextManager[None]) -> None: + self._lease = lease + self._closed = False + lease.__enter__() + + def close(self) -> None: + if self._closed: + return + self._closed = True + self._lease.__exit__(None, None, None) diff --git a/backend/services/patches/diffusion_stage_cache.py b/backend/services/patches/diffusion_stage_cache.py index 7882b813a..d60dae997 100644 --- a/backend/services/patches/diffusion_stage_cache.py +++ b/backend/services/patches/diffusion_stage_cache.py @@ -30,18 +30,15 @@ (``**kwargs: object, # noqa: ARG002`` in single_gpu_model_builder.py) -- confirmed inert for the path we cache, not assumed. -GENERATION-SCOPED, not session-scoped (found live on an RTX 5090): letting the -cache survive PAST the generation that built it collides with every other -component that builds fresh per call too (text encoder, VAE, upsampler, audio -decoder/vocoder) -- the next generation's text-encoder build then has to -coexist in VRAM with the still-resident transformer from the PREVIOUS -generation, instead of the transformer having already been freed by then. -Observed: a second generation's peak VRAM was reported at ~41.8 GB on a -31.82 GB card (Windows CUDA fell back to slow shared memory, backend liveness -probe failed, total generation time regressed to 143s -- worse than no cache -at all). Fix: ``handlers.generation_handler.GenerationHandler.start_generation`` -/``start_api_generation`` call :func:`evict` before marking a new generation as -running, so the cache never survives past the generation it was built for. +REUSE-SCOPED, not generation- or session-scoped: the cache exists only to bridge +the first and second identical diffusion stages. A live 720p Apple Silicon run +showed why the narrower lifetime matters: leaving the reused ~37 GiB transformer +resident through tiled VAE decode raised MPS driver memory to 46.7 GiB versus +44.5 GiB with the cache disabled, and left 35.4 GiB torch-allocated after output +encoding. The cache now evicts immediately when a hit's denoising context exits, +before decoder construction. The generation-start eviction remains a defensive +backstop for a pipeline that builds a cacheable first stage but never reaches an +identical second stage (cancellation, error, or a different stage configuration). NON-CACHEABLE TRANSITIONS also evict (found live on the same RTX 5090, IC-LoRA this time): IC-LoRA's ``use_lora_in_stage_2`` forces stage_2 onto the streaming @@ -90,7 +87,7 @@ produces a different key (cache miss, falls back to a normal rebuild), never a false hit. -Toggle: ``AppSettings.diffusion_stage_cache_enabled`` (default off, surfaced in +Toggle: ``AppSettings.diffusion_stage_cache_enabled`` (default on, surfaced in Settings next to Torch Compile). ``GenerationHandler.start_generation``/ ``start_api_generation`` push the live setting into :func:`set_enabled` on every generation, so flipping it in Settings takes effect on the next @@ -218,11 +215,18 @@ def evict() -> None: _evict_locked() -def _mark_free() -> None: - """Mark that a caller is done with a cached transformer it checked out.""" +def _release(*, evict_after_use: bool) -> None: + """Release one checkout and optionally retire the now-consumed cache entry. + + A hit means the cache fulfilled its only purpose: carrying one identical + transformer from the first diffusion stage into the second. Retire it while + leaving that second context so VAE/audio decode cannot overlap its weights. + """ global _in_use with _lock: _in_use = max(0, _in_use - 1) + if evict_after_use: + _evict_locked() _orig_transformer_ctx = DiffusionStage._transformer_ctx # noqa: SLF001 @@ -263,7 +267,7 @@ def _cached_transformer_ctx(self: DiffusionStage, **kwargs: object) -> Iterator[ try: yield model finally: - _mark_free() + _release(evict_after_use=hit) DiffusionStage._transformer_ctx = _cached_transformer_ctx # type: ignore[method-assign] # noqa: SLF001 diff --git a/backend/state/app_settings.py b/backend/state/app_settings.py index f3a3e987a..eb7666ffc 100644 --- a/backend/state/app_settings.py +++ b/backend/state/app_settings.py @@ -49,7 +49,7 @@ class SettingsPatchModel(SettingsBaseModel): class AppSettings(SettingsBaseModel): use_torch_compile: bool = False - diffusion_stage_cache_enabled: bool = False + diffusion_stage_cache_enabled: bool = True ltx_api_key: str = "" user_prefers_ltx_api_video_generations: bool = False fal_api_key: str = "" @@ -126,7 +126,7 @@ def _is_settings_model_annotation(annotation: object) -> TypeGuard[type[Settings class SettingsResponse(SettingsBaseModel): use_torch_compile: bool = False - diffusion_stage_cache_enabled: bool = False + diffusion_stage_cache_enabled: bool = True has_ltx_api_key: bool = False user_prefers_ltx_api_video_generations: bool = False has_fal_api_key: bool = False diff --git a/backend/state/app_state_types.py b/backend/state/app_state_types.py index e63c5f7f1..5d291a02a 100644 --- a/backend/state/app_state_types.py +++ b/backend/state/app_state_types.py @@ -3,7 +3,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import TYPE_CHECKING, NewType, Protocol +from typing import TYPE_CHECKING, Literal, NewType, Protocol from api_types import ModelCheckpointID from state.conditioning_cache import ConditioningCache @@ -122,6 +122,7 @@ class TextEncoderState: class VideoPipelineState: pipeline: FastVideoPipeline is_compiled: bool + runtime_engine: Literal["torch", "mlx"] = "torch" loras: tuple[tuple[str, float], ...] = field(default_factory=tuple) # gemma_root the pipeline's text encoder was built with. Part of the cache key: switching # text-encoding mode (API<->local) changes it, and a cached pipeline built for the other @@ -293,3 +294,6 @@ class AppState: # that raises on some validation path before ever reaching start_generation()/ # fail_generation() (both of which clear it) must not block every future generation forever. generation_starting_since: float | None = None + generation_starting_id: str | None = None + generation_starting_phase: str = "starting" + generation_start_cancelled: bool = False diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index be517c11a..0e15d0266 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -76,6 +76,7 @@ def test_state(tmp_path: Path, fake_services: FakeServices): ltx_api_client=fake_services.ltx_api_client, zit_api_client=fake_services.zit_api_client, fast_video_pipeline_class=type(fake_services.fast_video_pipeline), + mlx_fast_video_pipeline_class=type(fake_services.fast_video_pipeline), image_generation_pipeline_class=type(fake_services.image_generation_pipeline), ic_lora_pipeline_class=type(fake_services.ic_lora_pipeline), depth_processor_pipeline_class=type(fake_services.depth_processor_pipeline), diff --git a/backend/tests/test_diffusion_stage_cache.py b/backend/tests/test_diffusion_stage_cache.py index ee7e287f4..14a3386f1 100644 --- a/backend/tests/test_diffusion_stage_cache.py +++ b/backend/tests/test_diffusion_stage_cache.py @@ -84,6 +84,27 @@ def test_cache_hit_reuses_model_without_rebuilding() -> None: assert stage_1.build_count == 1 assert stage_2.build_count == 0, "stage_2 should reuse stage_1's cached build" assert model_1 is model_2 + assert model_1.freed_to == "meta", "a consumed cache hit must free before decoder construction" + assert dsc._cached_model is None + assert dsc._cached_key is None + + +def test_third_identical_stage_rebuilds_after_consumed_hit() -> None: + stage_1 = _FakeStage(_single_gpu_builder("ckpt.safetensors")) + stage_2 = _FakeStage(_single_gpu_builder("ckpt.safetensors")) + stage_3 = _FakeStage(_single_gpu_builder("ckpt.safetensors")) + + with dsc._cached_transformer_ctx(stage_1): + pass + with dsc._cached_transformer_ctx(stage_2): + pass + with dsc._cached_transformer_ctx(stage_3): + pass + + assert stage_1.build_count == 1 + assert stage_2.build_count == 0 + assert stage_3.build_count == 1 + assert dsc._cached_model is not None def test_cache_miss_on_different_checkpoint_evicts_old_model() -> None: diff --git a/backend/tests/test_generation.py b/backend/tests/test_generation.py index 7304d5a02..bd35dab71 100644 --- a/backend/tests/test_generation.py +++ b/backend/tests/test_generation.py @@ -178,11 +178,29 @@ def test_resolution_mapping_540p(self, client, test_state, fake_services, create r = client.post("/api/generate", json=_T2V_JSON) assert r.status_code == 200 + data = r.json() + assert data["resolved_width"] == 896 + assert data["resolved_height"] == 512 pipeline = fake_services.fast_video_pipeline call = pipeline.generate_calls[0] - assert call["width"] == 960 + assert call["width"] == 896 assert call["height"] == 512 + def test_resolution_mapping_540p_portrait(self, client, test_state, fake_services, create_fake_model_files): + create_fake_model_files() + _enable_local_text_encoding(test_state) + + r = client.post( + "/api/generate", + json={**_T2V_JSON, "aspectRatio": "9:16"}, + ) + assert r.status_code == 200 + assert r.json()["resolved_width"] == 512 + assert r.json()["resolved_height"] == 896 + call = fake_services.fast_video_pipeline.generate_calls[0] + assert call["width"] == 512 + assert call["height"] == 896 + def test_resolution_mapping_720p(self, client, test_state, fake_services, create_fake_model_files): create_fake_model_files() _enable_local_text_encoding(test_state) diff --git a/backend/tests/test_health.py b/backend/tests/test_health.py index 4608ae761..a78c9d76c 100644 --- a/backend/tests/test_health.py +++ b/backend/tests/test_health.py @@ -1,5 +1,12 @@ """Tests for /health and /api/gpu-info endpoints.""" +import json + +from services.fast_video_pipeline.mlx_profile import ( + begin_mlx_profile, + finish_mlx_profile, + reset_mlx_profile_for_tests, +) from state.app_state_types import GpuSlot, VideoPipelineState from tests.fakes.services import FakeFastVideoPipeline @@ -102,3 +109,43 @@ def test_returns_typed_snapshot(self, client): assert isinstance(data["available"], bool) for key in ("allocated_mib", "driver_mib", "recommended_max_mib"): assert data[key] is None or isinstance(data[key], int) + + +class TestRuntimeTelemetry: + def test_returns_process_system_and_lease_snapshot(self, client): + r = client.get("/api/runtime-telemetry") + assert r.status_code == 200 + data = r.json() + assert data["process_rss_mib"] > 0 + assert data["system_total_mib"] >= data["system_available_mib"] > 0 + assert data["local_metal_lease_status"] in {"idle", "waiting", "held"} + + def test_reports_terminal_mlx_profile_after_child_exit(self, client, tmp_path): + reset_mlx_profile_for_tests() + profile = tmp_path / "job.jsonl" + begin_mlx_profile(profile) + profile.write_text( + json.dumps( + { + "event": "run_end", + "timestamp_unix_seconds": 1002.0, + "mlx_active_gb": 0.25, + "mlx_cache_gb": 0.125, + "mlx_peak_gb": 6.0, + } + ) + + "\n", + encoding="utf-8", + ) + finish_mlx_profile(profile, "success") + try: + response = client.get("/api/runtime-telemetry") + assert response.status_code == 200 + data = response.json() + assert data["mlx_profile_status"] == "success" + assert data["mlx_active_mib"] == 256 + assert data["mlx_cache_mib"] == 128 + assert data["mlx_peak_mib"] == 6144 + assert data["mlx_profile_path"] == str(profile) + finally: + reset_mlx_profile_for_tests() diff --git a/backend/tests/test_mlx_fast_video_pipeline.py b/backend/tests/test_mlx_fast_video_pipeline.py new file mode 100644 index 000000000..e7bab0c43 --- /dev/null +++ b/backend/tests/test_mlx_fast_video_pipeline.py @@ -0,0 +1,136 @@ +"""Isolation and cancellation invariants for the MLX Fast sidecar.""" + +from __future__ import annotations + +import threading + +import pytest + +from runtime_config.mlx_runtime import MLXRuntimeDiscovery +from services.fast_video_pipeline import mlx_fast_video_pipeline as mlx_sidecar + + +class _BlockingProcess: + pid = 43210 + + def __init__(self, released: threading.Event) -> None: + self._released = released + self._return_code: int | None = None + + def poll(self) -> int | None: + return self._return_code + + def wait(self) -> int: + assert self._released.wait(timeout=2) + self._return_code = -9 + return self._return_code + + +def test_generation_uses_fresh_process_group_and_cancellation_kills_it(monkeypatch) -> None: + released = threading.Event() + started = threading.Event() + process = _BlockingProcess(released) + popen_calls: list[tuple[list[str], dict[str, object]]] = [] + killed: list[tuple[int, int]] = [] + + runtime = MLXRuntimeDiscovery( + command_prefix=("/opt/ltx-2-mlx",), + source="test", + version="test", + revision="test", + dirty=False, + core_version="test", + mlx_version="test", + compatible=True, + reason="test", + ) + monkeypatch.setattr(mlx_sidecar, "discover_mlx_runtime", lambda: runtime) + + def _popen(command, **kwargs): + popen_calls.append((command, kwargs)) + started.set() + return process + + def _killpg(pid: int, sig: int) -> None: + killed.append((pid, sig)) + released.set() + + monkeypatch.setattr(mlx_sidecar.subprocess, "Popen", _popen) + monkeypatch.setattr(mlx_sidecar.os, "killpg", _killpg) + pipeline = mlx_sidecar.MLXFastVideoPipeline( + model_source="test/model", + low_ram=True, + loras=[], + ) + errors: list[BaseException] = [] + + def _run() -> None: + try: + pipeline.generate( + prompt="test", + seed=1, + height=256, + width=384, + num_frames=9, + frame_rate=8, + images=[], + output_path="/tmp/test.mp4", + ) + except BaseException as exc: + errors.append(exc) + + thread = threading.Thread(target=_run) + thread.start() + assert started.wait(timeout=2) + assert mlx_sidecar.get_active_mlx_sidecar_pid() == process.pid + assert mlx_sidecar.cancel_active_mlx_sidecar() is True + thread.join(timeout=2) + + assert not thread.is_alive() + command, kwargs = popen_calls[0] + assert kwargs["start_new_session"] is True + assert kwargs["env"]["LTX2_VAE_DECODE_BUDGET_GB"] == "2" + assert "--auto-tiling" in command + assert command[command.index("--model-precision") + 1] == "bf16" + assert command[command.index("--profile-json") + 1].endswith(".jsonl") + assert killed == [(process.pid, mlx_sidecar.signal.SIGKILL)] + assert len(errors) == 1 + assert isinstance(errors[0], RuntimeError) + assert "cancelled" in str(errors[0]).lower() + assert mlx_sidecar.get_active_mlx_sidecar_pid() is None + + +def test_distilled_sidecar_rejects_non_grid_dimensions_before_launch(monkeypatch) -> None: + runtime = MLXRuntimeDiscovery( + command_prefix=("/opt/ltx-2-mlx",), + source="test", + version="test", + revision="test", + dirty=False, + core_version="test", + mlx_version="test", + compatible=True, + reason="test", + ) + monkeypatch.setattr(mlx_sidecar, "discover_mlx_runtime", lambda: runtime) + pipeline = mlx_sidecar.MLXFastVideoPipeline( + model_source="test/model", + low_ram=False, + loras=[], + ) + + with pytest.raises(ValueError, match="divisible by 64"): + pipeline.generate( + prompt="test", + seed=1, + height=544, + width=960, + num_frames=9, + frame_rate=8, + images=[], + output_path="/tmp/must-not-launch.mp4", + ) + + +def test_cancel_without_active_sidecar_is_noop() -> None: + assert mlx_sidecar.cancel_active_mlx_sidecar() is False diff --git a/backend/tests/test_mlx_profile.py b/backend/tests/test_mlx_profile.py new file mode 100644 index 000000000..092e88fc3 --- /dev/null +++ b/backend/tests/test_mlx_profile.py @@ -0,0 +1,106 @@ +"""Per-job MLX profile ingestion tests; no model or Metal work.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from services.fast_video_pipeline.mlx_profile import ( + begin_mlx_profile, + finish_mlx_profile, + get_mlx_profile_snapshot, + read_mlx_profile, + reset_mlx_profile_for_tests, +) + + +def _write_profile(path: Path) -> None: + records = [ + { + "schema_version": 2, + "event": "run_start", + "timestamp_unix_seconds": 1000.0, + "mlx_active_gb": 1.0, + "mlx_peak_gb": 2.0, + "mlx_cache_gb": 0.5, + "metadata": { + "runtime_version": "0.14.20.dev1", + "runtime_commit": "3171bac4ba901c0237faea2678c34034b37abc2a", + "device_name": "Apple Test", + "prompt": "must not be exposed", + }, + }, + { + "schema_version": 2, + "event": "phase_end", + "phase": "Decoding video + audio + muxing", + "timestamp_unix_seconds": 1001.0, + "mlx_active_gb": 0.25, + "mlx_peak_gb": 5.0, + "mlx_cache_gb": 0.125, + }, + { + "schema_version": 2, + "event": "run_end", + "timestamp_unix_seconds": 1002.0, + "mlx_active_gb": 0.1, + "mlx_peak_gb": 5.0, + "mlx_cache_gb": 0.05, + "observed_peak_mlx_gb": 6.0, + }, + ] + path.write_text("".join(json.dumps(row) + "\n" for row in records), encoding="utf-8") + + +def test_terminal_profile_retains_allocator_peak_phase_and_safe_identity(tmp_path: Path) -> None: + reset_mlx_profile_for_tests() + profile = tmp_path / "job.jsonl" + begin_mlx_profile(profile) + _write_profile(profile) + finish_mlx_profile(profile, "success") + + snapshot = get_mlx_profile_snapshot() + assert snapshot is not None + assert snapshot.status == "success" + assert snapshot.active_mib == 102 + assert snapshot.cache_mib == 51 + assert snapshot.peak_mib == 6144 + assert snapshot.phase == "Decoding video + audio + muxing" + assert snapshot.sampled_at is not None + assert snapshot.runtime_identity == { + "runtime_version": "0.14.20.dev1", + "runtime_commit": "3171bac4ba901c0237faea2678c34034b37abc2a", + "device_name": "Apple Test", + } + reset_mlx_profile_for_tests() + + +def test_incomplete_tail_and_nonfinite_values_cannot_break_telemetry(tmp_path: Path) -> None: + profile = tmp_path / "partial.jsonl" + profile.write_text( + '{"event":"run_start","timestamp_unix_seconds":Infinity,' + '"mlx_active_gb":Infinity,"mlx_peak_gb":2.0}\n' + '{"event":"phase_end","phase":"diffusion",', + encoding="utf-8", + ) + + snapshot = read_mlx_profile(profile) + assert snapshot is not None + assert snapshot.status == "running" + assert snapshot.sampled_at is None + assert snapshot.active_mib is None + assert snapshot.peak_mib == 2048 + + +def test_cancelled_job_keeps_synthetic_terminal_snapshot_when_child_cannot_flush(tmp_path: Path) -> None: + reset_mlx_profile_for_tests() + profile = tmp_path / "killed.jsonl" + begin_mlx_profile(profile) + finish_mlx_profile(profile, "cancelled") + + snapshot = get_mlx_profile_snapshot() + assert snapshot is not None + assert snapshot.status == "cancelled" + assert snapshot.profile_path == str(profile) + assert snapshot.peak_mib is None + reset_mlx_profile_for_tests() diff --git a/backend/tests/test_mlx_runtime.py b/backend/tests/test_mlx_runtime.py new file mode 100644 index 000000000..86586e3fd --- /dev/null +++ b/backend/tests/test_mlx_runtime.py @@ -0,0 +1,83 @@ +"""Portable MLX runtime discovery and exact-pin validation.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from runtime_config import mlx_runtime +from runtime_config.runtime_policy import MLX_RUNTIME_REVISION, MLX_RUNTIME_VERSION + + +class _Result: + def __init__(self, identity: dict[str, object]) -> None: + self.returncode = 0 + self.stdout = json.dumps(identity) + self.stderr = "" + + +def _identity(*, version: str = MLX_RUNTIME_VERSION, revision: str = MLX_RUNTIME_REVISION, dirty: bool = False): + return { + "runtime_version": version, + "runtime_commit": revision, + "runtime_dirty": dirty, + "core_version": "core-test", + "mlx_version": "mlx-test", + } + + +def test_explicit_python_precedes_default_and_requires_exact_clean_pin(monkeypatch, tmp_path: Path) -> None: + explicit = tmp_path / "explicit-python" + sibling = tmp_path / "sibling-python" + explicit.touch() + sibling.touch() + monkeypatch.setenv("LTX_MLX_PYTHON", str(explicit)) + monkeypatch.setattr(mlx_runtime.Path, "expanduser", lambda self: sibling if "video-models" in str(self) else self) + probed: list[str] = [] + + def _run(command, **_kwargs): + probed.append(command[0]) + return _Result(_identity()) + + monkeypatch.setattr(mlx_runtime.subprocess, "run", _run) + mlx_runtime.discover_mlx_runtime.cache_clear() + discovered = mlx_runtime.discover_mlx_runtime() + + assert discovered.compatible is True + assert discovered.command_prefix == (str(explicit), "-m", "ltx_pipelines_mlx.cli") + assert probed == [str(explicit)] + + +def test_incompatible_explicit_runtime_falls_through_to_exact_sibling(monkeypatch, tmp_path: Path) -> None: + explicit = tmp_path / "explicit-python" + sibling = tmp_path / "sibling-python" + explicit.touch() + sibling.touch() + monkeypatch.setenv("LTX_MLX_PYTHON", str(explicit)) + monkeypatch.setattr(mlx_runtime.Path, "expanduser", lambda self: sibling if "video-models" in str(self) else self) + + def _run(command, **_kwargs): + identity = _identity(version="wrong", revision="wrong") if command[0] == str(explicit) else _identity() + return _Result(identity) + + monkeypatch.setattr(mlx_runtime.subprocess, "run", _run) + mlx_runtime.discover_mlx_runtime.cache_clear() + discovered = mlx_runtime.discover_mlx_runtime() + + assert discovered.compatible is True + assert discovered.command_prefix == (str(sibling), "-m", "ltx_pipelines_mlx.cli") + + +def test_dirty_exact_runtime_is_not_eligible(monkeypatch, tmp_path: Path) -> None: + python = tmp_path / "python" + python.touch() + monkeypatch.setenv("LTX_MLX_PYTHON", str(python)) + monkeypatch.setattr(mlx_runtime.Path, "expanduser", lambda self: tmp_path / "missing" if "video-models" in str(self) else self) + monkeypatch.setattr(mlx_runtime.subprocess, "run", lambda *_args, **_kwargs: _Result(_identity(dirty=True))) + monkeypatch.setattr(mlx_runtime.shutil, "which", lambda _name: None) + mlx_runtime.discover_mlx_runtime.cache_clear() + discovered = mlx_runtime.discover_mlx_runtime() + + assert discovered.compatible is False + assert discovered.command_prefix is None + assert "dirty" in discovered.reason diff --git a/backend/tests/test_performance_hd_matrix.py b/backend/tests/test_performance_hd_matrix.py new file mode 100644 index 000000000..5f7cf3a24 --- /dev/null +++ b/backend/tests/test_performance_hd_matrix.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import ctypes +import fcntl +import json +from pathlib import Path + +from performance_runner import analyze_metal_trace +from performance_runner import hd_matrix as bench + + +def test_rusage_info_v2_buffer_matches_darwin_abi() -> None: + assert ctypes.sizeof(bench._RusageInfoV2) >= 160 + + +def test_matrix_covers_multiple_resolutions_durations_and_i2v() -> None: + cases = bench.default_matrix() + assert {case.resolution for case in cases} == {"540p", "720p", "1080p"} + assert {case.duration for case in cases} >= {5, 8} + assert any(case.image_conditioned for case in cases) + + +def test_distilled_payload_has_no_fake_teacache_or_tiling_switch() -> None: + payload = bench.default_matrix()[0].payload(None) + assert not any("tea" in key.lower() for key in payload) + assert not any("tile" in key.lower() for key in payload) + + +def test_shared_flock_treats_payload_as_untrusted_diagnostics(tmp_path: Path) -> None: + lock_path = tmp_path / "local-metal.lock" + lock_path.write_text(json.dumps({"schema": bench.LOCK_SCHEMA, "pid": 999999}), encoding="utf-8") + assert bench.probe_metal_lock(lock_path)["observed"] == "acquired" + with open(lock_path, "a+", encoding="utf-8") as owner: + fcntl.flock(owner.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + assert bench.probe_metal_lock(lock_path)["observed"] == "contended" + fcntl.flock(owner.fileno(), fcntl.LOCK_UN) + assert bench.probe_metal_lock(lock_path)["observed"] == "acquired" + + +def test_strict_validator_hard_fails_missing_authoritative_fields() -> None: + failures = bench.strict_failures({}) + assert "missing authoritative worker physical footprint" in failures + assert "missing explicit cleanup/post-cleanup telemetry" in failures + assert "shared Metal lease not held during local generation" in failures + + +def test_strict_validator_accepts_complete_torch_evidence() -> None: + row = { + "runtime_summary": {"peak_rss_gib": 1, "peak_physical_footprint_gib": 2}, + "runtime_policy": {"auto_fast_video_engine": "torch"}, + "progress_phases": ["inference"], "cleanup_evidence": [{"status": "cleanup"}], + "lease": {"running_probe": {"observed": "contended"}, "terminal_probe": {"observed": "acquired"}}, + "dimensions": { + "requested": {"resolution": "720p"}, + "resolved": {"width": 1280, "height": 704}, + "actual": {"width": 1280, "height": 704}, + }, + "hashes": {"recipe_sha256": "a", "prompt_sha256": "b", "source_sha256": "c", "repo_head": "d"}, + } + assert bench.strict_failures(row) == [] + + row["dimensions"]["actual"]["height"] = 512 + assert "resolved height 704 does not match actual height 512" in bench.strict_failures(row) + + +def test_metal_trace_analyzer_resolves_nested_process_references(tmp_path: Path) -> None: + export = tmp_path / "gpu_intervals.xml" + export.write_text( + """ + + 1005000 + Compute + 00 + + 42 + + + + 3 + + """, + encoding="utf-8", + ) + report = analyze_metal_trace.analyze_gpu_intervals(export, 42) + assert report["interval_count"] == 1 + assert report["channels"] == [{"channel": "Compute", "count": 1, "sum_interval_seconds": 0.000005}] + assert report["top_dispatches"][0]["command_buffer_id"] == "0x1" diff --git a/backend/tests/test_runtime_policy.py b/backend/tests/test_runtime_policy.py index b36828db9..c8f9faa86 100644 --- a/backend/tests/test_runtime_policy.py +++ b/backend/tests/test_runtime_policy.py @@ -8,7 +8,11 @@ def test_runtime_policy_true(client, test_state): response = client.get("/api/runtime-policy") assert response.status_code == 200 - assert response.json() == {"force_api_generations": True} + data = response.json() + assert data["force_api_generations"] is True + assert data["auto_fast_video_engine"] == "cloud" + assert data["execution_mode"] == "unsupported" + assert data["provenance"] def test_runtime_policy_false(client, test_state): @@ -16,4 +20,8 @@ def test_runtime_policy_false(client, test_state): response = client.get("/api/runtime-policy") assert response.status_code == 200 - assert response.json() == {"force_api_generations": False} + data = response.json() + assert data["force_api_generations"] is False + assert data["auto_fast_video_engine"] == "torch" + assert data["execution_mode"] == "eager" + assert data["capability_engines"]["retake"] == "torch" diff --git a/backend/tests/test_runtime_policy_decision.py b/backend/tests/test_runtime_policy_decision.py index 5371a5425..8961f11a3 100644 --- a/backend/tests/test_runtime_policy_decision.py +++ b/backend/tests/test_runtime_policy_decision.py @@ -5,6 +5,8 @@ import pytest from runtime_config.runtime_policy import ( + decide_fast_video_engine, + decide_fast_video_execution_mode, decide_local_generation_mode, streaming_prefetch_count_for_mode, ) @@ -121,3 +123,69 @@ def test_streaming_prefetch_count_for_streaming_mode_is_two() -> None: def test_streaming_prefetch_count_for_unsupported_asserts() -> None: with pytest.raises(AssertionError): streaming_prefetch_count_for_mode("unsupported") + + +def test_fast_auto_uses_mlx_only_for_cached_local_text_path() -> None: + decision = decide_fast_video_engine( + preference="auto", + mlx_runtime_eligible=True, + mlx_model_cached=True, + use_local_text_encoding=True, + ) + assert decision.engine == "mlx" + + +def test_fast_auto_preserves_prepared_embeddings_on_torch() -> None: + decision = decide_fast_video_engine( + preference="auto", + mlx_runtime_eligible=True, + mlx_model_cached=True, + use_local_text_encoding=False, + ) + assert decision.engine == "torch" + assert "embeddings" in decision.reason + + +def test_fast_auto_does_not_trigger_hidden_model_download() -> None: + decision = decide_fast_video_engine( + preference="auto", + mlx_runtime_eligible=True, + mlx_model_cached=False, + use_local_text_encoding=True, + ) + assert decision.engine == "torch" + assert "not cached" in decision.reason + + +def test_explicit_mlx_can_use_uncached_bf16_model() -> None: + decision = decide_fast_video_engine( + preference="mlx", + mlx_runtime_eligible=True, + mlx_model_cached=False, + use_local_text_encoding=False, + ) + assert decision.engine == "mlx" + assert "download" in decision.reason + + +def test_mlx_bf16_uses_low_ram_below_eager_floor() -> None: + assert ( + decide_fast_video_execution_mode("mlx", "streaming_models_loading", 48) + == "low_ram" + ) + assert ( + decide_fast_video_execution_mode("mlx", "streaming_models_loading", 64) + == "eager" + ) + + +def test_fast_auto_never_selects_unqualified_q8() -> None: + decision = decide_fast_video_engine( + preference="auto", + mlx_runtime_eligible=True, + mlx_model_cached=True, + use_local_text_encoding=True, + mlx_quality_qualified=False, + ) + assert decision.engine == "torch" + assert "never auto-selected" in decision.reason diff --git a/backend/tests/test_settings.py b/backend/tests/test_settings.py index 6df0486fe..c38417136 100644 --- a/backend/tests/test_settings.py +++ b/backend/tests/test_settings.py @@ -181,6 +181,7 @@ def test_models_dir_persists_and_loads(self, client, test_state, default_app_set ltx_api_client=fake_services.ltx_api_client, zit_api_client=fake_services.zit_api_client, fast_video_pipeline_class=type(fake_services.fast_video_pipeline), + mlx_fast_video_pipeline_class=type(fake_services.fast_video_pipeline), image_generation_pipeline_class=type(fake_services.image_generation_pipeline), ic_lora_pipeline_class=type(fake_services.ic_lora_pipeline), depth_processor_pipeline_class=type(fake_services.depth_processor_pipeline), @@ -209,6 +210,7 @@ def _new_state(self, test_state, default_app_settings): ltx_api_client=fake_services.ltx_api_client, zit_api_client=fake_services.zit_api_client, fast_video_pipeline_class=type(fake_services.fast_video_pipeline), + mlx_fast_video_pipeline_class=type(fake_services.fast_video_pipeline), image_generation_pipeline_class=type(fake_services.image_generation_pipeline), ic_lora_pipeline_class=type(fake_services.ic_lora_pipeline), depth_processor_pipeline_class=type(fake_services.depth_processor_pipeline), diff --git a/backend/tests/test_video_resolution.py b/backend/tests/test_video_resolution.py index 67a25eff6..560b470d8 100644 --- a/backend/tests/test_video_resolution.py +++ b/backend/tests/test_video_resolution.py @@ -5,7 +5,30 @@ import pytest from _routes._errors import HTTPError -from handlers.video_resolution import correct_frame_count, correct_resolution +from handlers.video_resolution import ( + correct_frame_count, + correct_resolution, + resolve_fast_video_dimensions, +) + + +def test_fast_540p_uses_aspect_aware_two_stage_grid_under_budget(): + width, height = resolve_fast_video_dimensions("540p", "16:9") + assert (width, height) == (896, 512) + assert width % 64 == 0 and height % 64 == 0 + assert width * height <= 960 * 544 + assert abs(width / height - 16 / 9) < abs(960 / 512 - 16 / 9) + + +def test_fast_540p_portrait_swaps_the_qualified_pair(): + assert resolve_fast_video_dimensions("540p", "9:16") == (512, 896) + + +def test_fast_resolution_rejects_unknown_catalog_or_aspect(): + with pytest.raises(ValueError): + resolve_fast_video_dimensions("1440p", "16:9") + with pytest.raises(ValueError): + resolve_fast_video_dimensions("540p", "4:3") def test_correct_resolution_snaps_height_down_to_div32(): diff --git a/frontend/components/SettingsModal.tsx b/frontend/components/SettingsModal.tsx index 39de26c5b..f5995237c 100644 --- a/frontend/components/SettingsModal.tsx +++ b/frontend/components/SettingsModal.tsx @@ -1,4 +1,4 @@ -import { AlertCircle, Check, Download, Film, Folder, HardDrive, Info, KeyRound, Settings, Sparkles, X, Zap } from 'lucide-react' +import { Activity, AlertCircle, Check, Download, Film, Folder, HardDrive, Info, KeyRound, Settings, Sparkles, X, Zap } from 'lucide-react' import React, { useEffect, useMemo, useRef, useState } from 'react' import { Button } from './ui/button' import { BaseModelSection } from './settings/BaseModelSection' @@ -15,7 +15,7 @@ interface SettingsModalProps { initialTab?: TabId } -type TabId = 'general' | 'models' | 'apiKeys' | 'promptEnhancer' | 'about' +type TabId = 'general' | 'models' | 'apiKeys' | 'promptEnhancer' | 'performance' | 'about' /** Focuses an API Keys tab input once the modal has switched to that tab. * Shared by the LTX and FAL key inputs — each call gets its own ref/pending state. */ @@ -93,7 +93,17 @@ function SettingToggle({ title, description, enabled, onToggle, statusOn, status } export function SettingsModal({ isOpen, onClose, initialTab }: SettingsModalProps) { - const { settings, updateSettings, saveLtxApiKey, saveFalApiKey, saveGeminiApiKey, forceApiGenerations, cudaAvailable } = useAppSettings() + const { + settings, + updateSettings, + saveLtxApiKey, + saveFalApiKey, + saveGeminiApiKey, + forceApiGenerations, + cudaAvailable, + mpsAvailable, + runtimePolicy, + } = useAppSettings() const onSettingsChange = (next: AppSettings) => updateSettings(next) const [activeTab, setActiveTab] = useState('general') const ltxApiKey = useApiKeyFocus(isOpen, activeTab, setActiveTab) @@ -124,6 +134,7 @@ export function SettingsModal({ isOpen, onClose, initialTab }: SettingsModalProp const [showModelLicense, setShowModelLicense] = useState(false) const [analyticsEnabled, setAnalyticsEnabled] = useState(false) const [projectAssetsPath, setProjectAssetsPath] = useState('') + const [runtimeTelemetry, setRuntimeTelemetry] = useState | null>(null) // Sync active tab with initialTab prop when modal opens useEffect(() => { @@ -146,6 +157,21 @@ export function SettingsModal({ isOpen, onClose, initialTab }: SettingsModalProp window.electronAPI.getAppInfo().then(info => setAppVersion(info.version)).catch(() => {}) }, [activeTab, appVersion]) + useEffect(() => { + if (!isOpen || activeTab !== 'performance') return + let cancelled = false + const poll = async () => { + const result = await ApiClient.getRuntimeTelemetry() + if (result.ok && !cancelled) setRuntimeTelemetry(result.data) + } + void poll() + const interval = window.setInterval(() => { void poll() }, 1000) + return () => { + cancelled = true + window.clearInterval(interval) + } + }, [activeTab, isOpen]) + // Fetch analytics state when modal opens useEffect(() => { if (!isOpen) return @@ -325,6 +351,7 @@ export function SettingsModal({ isOpen, onClose, initialTab }: SettingsModalProp ...(!forceApiGenerations ? [{ id: 'models' as TabId, label: 'Models', icon: HardDrive }] : []), { id: 'apiKeys' as TabId, label: 'API Keys', icon: KeyRound }, { id: 'promptEnhancer' as TabId, label: 'Prompt Enhancer', icon: Sparkles }, + { id: 'performance' as TabId, label: 'Performance', icon: Activity }, { id: 'about' as TabId, label: 'About', icon: Info }, ] @@ -433,10 +460,10 @@ export function SettingsModal({ isOpen, onClose, initialTab }: SettingsModalProp
- Generate With API + Use LTX API Models

- Use LTX API for video generation when an LTX API key is configured. + Unlocks LTX-2.3 Fast and Pro in the model selector. Local generation supports Fast only.

)} - {/* Torch Compile + Diffusion Stage Cache -- CUDA only, no-op on MPS/CPU */} + {/* torch.compile remains CUDA-only. */} {cudaAvailable && ( )} - {cudaAvailable && ( + {(cudaAvailable || mpsAvailable) && ( Reuses an already-built transformer across stage 1/stage 2 within one generation - instead of reloading it from disk twice. Experimental: only - applies on high-VRAM cards (32GB+); no effect otherwise.} + instead of reloading it from disk twice, then releases it before VAE decode. Experimental: applies to the official Torch runtime; MLX manages stages independently.} enabled={settings.diffusionStageCacheEnabled} onToggle={handleToggleDiffusionStageCache} statusOn="Skipping redundant transformer reloads" @@ -1139,6 +1165,144 @@ export function SettingsModal({ isOpen, onClose, initialTab }: SettingsModalProp )} + {activeTab === 'performance' && ( +
+
+
+
+

Local runtime policy

+

+ Capability-aware selection keeps feature-rich requests on Torch and uses MLX only where parity is qualified. +

+
+ + {runtimePolicy?.auto_fast_video_engine ?? '...'} + +
+ {runtimePolicy && ( + <> +
+
+
Preference
+
{runtimePolicy.fast_video_engine_preference}
+
+
+
Memory mode
+
+ {runtimePolicy.execution_mode}{runtimePolicy.automatic_tiling ? ' + auto tiling' : ''} +
+
+
+

{runtimePolicy.auto_selection_reason}

+
+ Model: {runtimePolicy.mlx_model_source} ({runtimePolicy.mlx_model_variant}) +
+ {runtimePolicy.quality_warning && ( +
+ {runtimePolicy.quality_warning} +
+ )} + + )} +
+ +
+

Live / last-job memory

+
+ {[ + ['Process RSS', runtimeTelemetry?.process_rss_mib], + ['System available', runtimeTelemetry?.system_available_mib], + ['MLX active', runtimeTelemetry?.mlx_active_mib], + ['MLX cache', runtimeTelemetry?.mlx_cache_mib], + ['MLX peak', runtimeTelemetry?.mlx_peak_mib], + ['MPS allocated', runtimeTelemetry?.mps_allocated_mib], + ['MPS driver', runtimeTelemetry?.mps_driver_mib], + ].map(([label, value]) => ( +
+
{label}
+
+ {typeof value === 'number' ? `${value.toLocaleString()} MiB` : '—'} +
+
+ ))} +
+ {runtimeTelemetry?.mlx_profile_status && ( +
+
+ MLX profile + {runtimeTelemetry.mlx_profile_status} +
+
+ Phase: {runtimeTelemetry.mlx_profile_phase ?? 'initializing'} +
+ {runtimeTelemetry.mlx_runtime_identity && ( +
+ Runtime {String(runtimeTelemetry.mlx_runtime_identity.runtime_version ?? 'unknown')} + {' · '}{String(runtimeTelemetry.mlx_runtime_identity.runtime_commit ?? 'unknown').slice(0, 12)} + {' · '}{String(runtimeTelemetry.mlx_runtime_identity.device_name ?? 'unknown device')} +
+ )} + {runtimeTelemetry.mlx_profile_path && ( +
{runtimeTelemetry.mlx_profile_path}
+ )} +
+ )} +
+ +
+
+

Shared local accelerator

+ + {runtimeTelemetry?.local_metal_lease_status ?? 'idle'} + +
+

+ LTX apps coordinate through one advisory Metal lease so two heavy local renders cannot exhaust unified memory. +

+ {runtimeTelemetry?.local_metal_lease_reason && ( +

{runtimeTelemetry.local_metal_lease_reason}

+ )} + {runtimeTelemetry?.local_metal_lease_owner && ( +

+ Owner: {String(runtimeTelemetry.local_metal_lease_owner.product ?? 'unknown')} + {' '}· PID {String(runtimeTelemetry.local_metal_lease_owner.pid ?? '?')} + {' '}· {runtimeTelemetry.local_metal_lease_waited_seconds.toFixed(1)}s +

+ )} +
+ + {runtimePolicy && ( +
+

Capability routing

+
+ {Object.entries(runtimePolicy.capability_engines).map(([capability, engine]) => ( +
+ {capability.replace(/_/g, ' ')} + {engine} +
+ ))} +
+

Runtime provenance

+ {runtimePolicy.provenance.map(item => ( +
+
+ {item.component} + {item.version} +
+ {item.revision &&
{item.revision}
} +
+ ))} +
+ )} +
+ )} + {activeTab === 'about' && ( <> {showModelLicense ? ( diff --git a/frontend/contexts/AppSettingsContext.tsx b/frontend/contexts/AppSettingsContext.tsx index 6833b62ac..cae1d41bd 100644 --- a/frontend/contexts/AppSettingsContext.tsx +++ b/frontend/contexts/AppSettingsContext.tsx @@ -57,6 +57,8 @@ interface AppSettingsContextValue { shouldVideoGenerateWithLtxApi: boolean shouldImageGenerateWithFalApi: boolean cudaAvailable: boolean + mpsAvailable: boolean + runtimePolicy: RuntimePolicyPayload | null } const AppSettingsContext = createContext(null) @@ -102,6 +104,8 @@ export function AppSettingsProvider({ children }: { children: ReactNode }) { const [runtimePolicyLoaded, setRuntimePolicyLoaded] = useState(false) const [forceApiGenerations, setForceApiGenerations] = useState(true) const [cudaAvailable, setCudaAvailable] = useState(false) + const [mpsAvailable, setMpsAvailable] = useState(false) + const [runtimePolicy, setRuntimePolicy] = useState(null) const [backendProcessStatus, setBackendProcessStatus] = useState(null) useEffect(() => { @@ -122,6 +126,7 @@ export function AppSettingsProvider({ children }: { children: ReactNode }) { } const payload = result.data as RuntimePolicyPayload + setRuntimePolicy(payload) if (typeof payload.force_api_generations !== 'boolean') { if (!cancelled) { setForceApiGenerations(true) @@ -153,6 +158,7 @@ export function AppSettingsProvider({ children }: { children: ReactNode }) { const payload = result.data as GpuInfoPayload setCudaAvailable(Boolean(payload.cuda_available)) + setMpsAvailable(Boolean(payload.mps_available)) } void fetchGpuInfo() @@ -291,8 +297,10 @@ export function AppSettingsProvider({ children }: { children: ReactNode }) { shouldVideoGenerateWithLtxApi, shouldImageGenerateWithFalApi, cudaAvailable, + mpsAvailable, + runtimePolicy, }), - [cudaAvailable, forceApiGenerations, isLoaded, refreshSettings, runtimePolicyLoaded, saveFalApiKey, saveGeminiApiKey, saveLtxApiKey, settings, shouldVideoGenerateWithLtxApi, shouldImageGenerateWithFalApi, updateSettings], + [cudaAvailable, forceApiGenerations, isLoaded, mpsAvailable, refreshSettings, runtimePolicy, runtimePolicyLoaded, saveFalApiKey, saveGeminiApiKey, saveLtxApiKey, settings, shouldVideoGenerateWithLtxApi, shouldImageGenerateWithFalApi, updateSettings], ) return {children} diff --git a/frontend/contexts/ProjectContext.tsx b/frontend/contexts/ProjectContext.tsx index af01554de..47b286a73 100644 --- a/frontend/contexts/ProjectContext.tsx +++ b/frontend/contexts/ProjectContext.tsx @@ -179,11 +179,14 @@ export function ProjectProvider({ children }: { children: React.ReactNode }) { createdAt: Date.now(), } - mutateProject(projectId, project => ({ + const persistedProject = mutateProject(projectId, project => ({ ...project, assets: [newAsset, ...project.assets], updatedAt: Date.now(), })) + if (!persistedProject) { + throw new Error(`Cannot add asset: project ${projectId} was not found`) + } return newAsset }, [mutateProject]) diff --git a/frontend/generated/backend-openapi.json b/frontend/generated/backend-openapi.json index f439beab7..d746c4328 100644 --- a/frontend/generated/backend-openapi.json +++ b/frontend/generated/backend-openapi.json @@ -892,6 +892,28 @@ }, "GenerateVideoCompleteResponse": { "properties": { + "resolved_height": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Resolved Height" + }, + "resolved_width": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Resolved Width" + }, "status": { "const": "complete", "title": "Status", @@ -3403,17 +3425,358 @@ }, "RuntimePolicyResponse": { "properties": { + "auto_fast_video_engine": { + "enum": [ + "torch", + "mlx", + "cloud" + ], + "title": "Auto Fast Video Engine", + "type": "string" + }, + "auto_selection_reason": { + "title": "Auto Selection Reason", + "type": "string" + }, + "automatic_tiling": { + "title": "Automatic Tiling", + "type": "boolean" + }, + "capability_engines": { + "additionalProperties": { + "enum": [ + "torch", + "mlx", + "cloud" + ], + "type": "string" + }, + "title": "Capability Engines", + "type": "object" + }, + "execution_mode": { + "enum": [ + "eager", + "low_ram", + "unsupported" + ], + "title": "Execution Mode", + "type": "string" + }, + "fast_video_engine_preference": { + "enum": [ + "auto", + "torch", + "mlx" + ], + "title": "Fast Video Engine Preference", + "type": "string" + }, "force_api_generations": { "title": "Force Api Generations", "type": "boolean" + }, + "mlx_model_source": { + "title": "Mlx Model Source", + "type": "string" + }, + "mlx_model_variant": { + "enum": [ + "bf16", + "q8" + ], + "title": "Mlx Model Variant", + "type": "string" + }, + "provenance": { + "items": { + "$ref": "#/components/schemas/RuntimeProvenanceItem" + }, + "title": "Provenance", + "type": "array" + }, + "quality_warning": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Quality Warning" } }, "required": [ - "force_api_generations" + "force_api_generations", + "fast_video_engine_preference", + "auto_fast_video_engine", + "auto_selection_reason", + "execution_mode", + "automatic_tiling", + "mlx_model_source", + "mlx_model_variant", + "capability_engines", + "provenance" ], "title": "RuntimePolicyResponse", "type": "object" }, + "RuntimeProvenanceItem": { + "properties": { + "component": { + "title": "Component", + "type": "string" + }, + "revision": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Revision" + }, + "source": { + "title": "Source", + "type": "string" + }, + "version": { + "title": "Version", + "type": "string" + } + }, + "required": [ + "component", + "version", + "source" + ], + "title": "RuntimeProvenanceItem", + "type": "object" + }, + "RuntimeTelemetryResponse": { + "properties": { + "active_engine": { + "anyOf": [ + { + "enum": [ + "torch", + "mlx", + "cloud" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Active Engine" + }, + "active_pipeline": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Active Pipeline" + }, + "local_metal_lease_owner": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Local Metal Lease Owner" + }, + "local_metal_lease_reason": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Local Metal Lease Reason" + }, + "local_metal_lease_status": { + "enum": [ + "idle", + "waiting", + "held" + ], + "title": "Local Metal Lease Status", + "type": "string" + }, + "local_metal_lease_waited_seconds": { + "default": 0.0, + "title": "Local Metal Lease Waited Seconds", + "type": "number" + }, + "mlx_active_mib": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Mlx Active Mib" + }, + "mlx_cache_mib": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Mlx Cache Mib" + }, + "mlx_peak_mib": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Mlx Peak Mib" + }, + "mlx_profile_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Mlx Profile Path" + }, + "mlx_profile_phase": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Mlx Profile Phase" + }, + "mlx_profile_sampled_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Mlx Profile Sampled At" + }, + "mlx_profile_status": { + "anyOf": [ + { + "enum": [ + "running", + "success", + "error", + "cancelled" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Mlx Profile Status" + }, + "mlx_runtime_identity": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Mlx Runtime Identity" + }, + "mps_allocated_mib": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Mps Allocated Mib" + }, + "mps_driver_mib": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Mps Driver Mib" + }, + "mps_recommended_max_mib": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Mps Recommended Max Mib" + }, + "process_rss_mib": { + "title": "Process Rss Mib", + "type": "integer" + }, + "sampled_at": { + "title": "Sampled At", + "type": "string" + }, + "system_available_mib": { + "title": "System Available Mib", + "type": "integer" + }, + "system_total_mib": { + "title": "System Total Mib", + "type": "integer" + } + }, + "required": [ + "sampled_at", + "process_rss_mib", + "system_total_mib", + "system_available_mib", + "local_metal_lease_status" + ], + "title": "RuntimeTelemetryResponse", + "type": "object" + }, "SetActiveLtxModelRequest": { "properties": { "model_id": { @@ -3449,7 +3812,7 @@ "title": "Activeltxmodelid" }, "diffusionStageCacheEnabled": { - "default": false, + "default": true, "title": "Diffusionstagecacheenabled", "type": "boolean" }, @@ -5496,6 +5859,47 @@ ] } }, + "/api/runtime-telemetry": { + "get": { + "operationId": "route_runtime_telemetry_api_runtime_telemetry_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RuntimeTelemetryResponse" + } + } + }, + "description": "Successful Response" + }, + "4XX": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPErrorResponse" + } + } + }, + "description": "Client Error" + }, + "5XX": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPErrorResponse" + } + } + }, + "description": "Server Error" + } + }, + "summary": "Route Runtime Telemetry", + "tags": [ + "health" + ] + } + }, "/api/settings": { "get": { "operationId": "route_get_settings_api_settings_get", diff --git a/frontend/generated/backend-openapi.ts b/frontend/generated/backend-openapi.ts index 1c29d9fbb..f206cdaf1 100644 --- a/frontend/generated/backend-openapi.ts +++ b/frontend/generated/backend-openapi.ts @@ -638,6 +638,23 @@ export interface paths { patch?: never; trace?: never; }; + "/api/runtime-telemetry": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Route Runtime Telemetry */ + get: operations["route_runtime_telemetry_api_runtime_telemetry_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/settings": { parameters: { query?: never; @@ -1017,6 +1034,10 @@ export interface components { }; /** GenerateVideoCompleteResponse */ GenerateVideoCompleteResponse: { + /** Resolved Height */ + resolved_height?: number | null; + /** Resolved Width */ + resolved_width?: number | null; /** * Status * @constant @@ -1929,8 +1950,108 @@ export interface components { }; /** RuntimePolicyResponse */ RuntimePolicyResponse: { + /** + * Auto Fast Video Engine + * @enum {string} + */ + auto_fast_video_engine: "torch" | "mlx" | "cloud"; + /** Auto Selection Reason */ + auto_selection_reason: string; + /** Automatic Tiling */ + automatic_tiling: boolean; + /** Capability Engines */ + capability_engines: { + [key: string]: "torch" | "mlx" | "cloud"; + }; + /** + * Execution Mode + * @enum {string} + */ + execution_mode: "eager" | "low_ram" | "unsupported"; + /** + * Fast Video Engine Preference + * @enum {string} + */ + fast_video_engine_preference: "auto" | "torch" | "mlx"; /** Force Api Generations */ force_api_generations: boolean; + /** Mlx Model Source */ + mlx_model_source: string; + /** + * Mlx Model Variant + * @enum {string} + */ + mlx_model_variant: "bf16" | "q8"; + /** Provenance */ + provenance: components["schemas"]["RuntimeProvenanceItem"][]; + /** Quality Warning */ + quality_warning?: string | null; + }; + /** RuntimeProvenanceItem */ + RuntimeProvenanceItem: { + /** Component */ + component: string; + /** Revision */ + revision?: string | null; + /** Source */ + source: string; + /** Version */ + version: string; + }; + /** RuntimeTelemetryResponse */ + RuntimeTelemetryResponse: { + /** Active Engine */ + active_engine?: ("torch" | "mlx" | "cloud") | null; + /** Active Pipeline */ + active_pipeline?: string | null; + /** Local Metal Lease Owner */ + local_metal_lease_owner?: { + [key: string]: unknown; + } | null; + /** Local Metal Lease Reason */ + local_metal_lease_reason?: string | null; + /** + * Local Metal Lease Status + * @enum {string} + */ + local_metal_lease_status: "idle" | "waiting" | "held"; + /** + * Local Metal Lease Waited Seconds + * @default 0 + */ + local_metal_lease_waited_seconds: number; + /** Mlx Active Mib */ + mlx_active_mib?: number | null; + /** Mlx Cache Mib */ + mlx_cache_mib?: number | null; + /** Mlx Peak Mib */ + mlx_peak_mib?: number | null; + /** Mlx Profile Path */ + mlx_profile_path?: string | null; + /** Mlx Profile Phase */ + mlx_profile_phase?: string | null; + /** Mlx Profile Sampled At */ + mlx_profile_sampled_at?: string | null; + /** Mlx Profile Status */ + mlx_profile_status?: ("running" | "success" | "error" | "cancelled") | null; + /** Mlx Runtime Identity */ + mlx_runtime_identity?: { + [key: string]: unknown; + } | null; + /** Mps Allocated Mib */ + mps_allocated_mib?: number | null; + /** Mps Driver Mib */ + mps_driver_mib?: number | null; + /** Mps Recommended Max Mib */ + mps_recommended_max_mib?: number | null; + /** Process Rss Mib */ + process_rss_mib: number; + /** Sampled At */ + sampled_at: string; + /** System Available Mib */ + system_available_mib: number; + /** System Total Mib */ + system_total_mib: number; }; /** SetActiveLtxModelRequest */ SetActiveLtxModelRequest: { @@ -1946,7 +2067,7 @@ export interface components { activeLtxModelId?: ("ltx-2.3-22b-distilled-1.1" | "ltx-2.3-22b-distilled") | null; /** * Diffusionstagecacheenabled - * @default false + * @default true */ diffusionStageCacheEnabled: boolean; /** @@ -3537,6 +3658,44 @@ export interface operations { }; }; }; + route_runtime_telemetry_api_runtime_telemetry_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RuntimeTelemetryResponse"]; + }; + }; + /** @description Client Error */ + "4XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPErrorResponse"]; + }; + }; + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPErrorResponse"]; + }; + }; + }; + }; route_get_settings_api_settings_get: { parameters: { query?: never; diff --git a/frontend/lib/api-client.ts b/frontend/lib/api-client.ts index 86a13458e..06f4f53ef 100644 --- a/frontend/lib/api-client.ts +++ b/frontend/lib/api-client.ts @@ -359,6 +359,8 @@ export class ApiClient { static getRuntimePolicy = makeEndpointClient('/api/runtime-policy', 'get') + static getRuntimeTelemetry = makeEndpointClient('/api/runtime-telemetry', 'get') + static getGpuInfo = makeEndpointClient('/api/gpu-info', 'get') static getSettings = makeEndpointClient('/api/settings', 'get') diff --git a/frontend/views/GenSpace.tsx b/frontend/views/GenSpace.tsx index 2f8de5750..85604ae32 100644 --- a/frontend/views/GenSpace.tsx +++ b/frontend/views/GenSpace.tsx @@ -914,10 +914,22 @@ function PromptBar({ title="MODEL" value={resolvedVideoOptions.selectedModel ?? settings.model} onChange={(v) => onSettingsChange({ ...settings, model: v })} - options={resolvedVideoOptions.modelOptions.map((item) => ({ - value: item.pipeline, - label: item.spec.display_name, - }))} + options={[ + ...resolvedVideoOptions.modelOptions.map((item) => ({ + value: item.pipeline, + label: item.spec.display_name, + })), + ...( + isLocalMode && !resolvedVideoOptions.modelOptions.some((item) => item.pipeline === 'pro') + ? [{ + value: 'pro', + label: 'LTX-2.3 Pro (API)', + disabled: true, + tooltip: 'Enable LTX API models in Settings → General', + }] + : [] + ), + ]} trigger={ <> @@ -1446,6 +1458,7 @@ export function GenSpace() { prompt: string input: { videoPath: string; direction: ExtendDirection; duration: number; videoDuration: number } } | null>(null) + const persistingExtendResultRef = useRef(null) const [retakeInitial, setRetakeInitial] = useState<{ videoPath: string | null duration?: number @@ -1850,60 +1863,69 @@ export function GenSpace() { })() }, [retakeResult, isRetaking, currentProjectId, activeProject?.assets, activeRetakeSource, addAsset, addTakeToAsset, setPendingRetakeUpdate, resetRetake]) - // When extend completes, save the longer video as a new asset. + // When extend completes, save the longer video as a new asset. Keep the recovery marker until + // the project write succeeds: copying/thumbnails can finish before a later storage error, and + // dropping the marker earlier would leave a valid output stranded with no retry path. useEffect(() => { if (!extendResult || !currentProjectId || isExtending) return const submission = extendSubmissionRef.current if (!submission) return - extendSubmissionRef.current = null - localStorage.removeItem(GENERATION_RECOVERY_KEY) + const generationKey = extendResult.videoPath + if (persistingExtendResultRef.current === generationKey) return + persistingExtendResultRef.current = generationKey ;(async () => { - const usedPrompt = submission.prompt - const usedInput = submission.input - const copied = await addVisualAssetToProject(extendResult.videoPath, currentProjectId, 'video') - if (!copied) { - logger.error('Could not persist extend result to project storage') - setLocalError(createLocalGenerationError('Failed to save extend output to project storage.')) - resetExtend() - return - } + try { + const usedPrompt = submission.prompt + const usedInput = submission.input + const copied = await addVisualAssetToProject(extendResult.videoPath, currentProjectId, 'video') + if (!copied) throw new Error('Could not persist extend result to project storage') - addAsset(currentProjectId, { - type: 'video', - path: copied.path, - bigThumbnailPath: copied.bigThumbnailPath, - smallThumbnailPath: copied.smallThumbnailPath, - width: copied.width, - height: copied.height, - prompt: usedPrompt, - resolution: '', - duration: usedInput.videoDuration + usedInput.duration, - generationParams: { - mode: 'extend', - prompt: usedPrompt, - model: 'pro', - duration: usedInput.videoDuration + usedInput.duration, - resolution: '', - fps: 24, - audio: true, - cameraMotion: 'none', - extendVideoPath: copied.path, - extendDuration: usedInput.duration, - extendDirection: usedInput.direction, - }, - takes: [{ + addAsset(currentProjectId, { + type: 'video', path: copied.path, bigThumbnailPath: copied.bigThumbnailPath, smallThumbnailPath: copied.smallThumbnailPath, width: copied.width, height: copied.height, - createdAt: Date.now(), - }], - activeTakeIndex: 0, - }) - setMode('video') - resetExtend() + prompt: usedPrompt, + resolution: '', + duration: usedInput.videoDuration + usedInput.duration, + generationParams: { + mode: 'extend', + prompt: usedPrompt, + model: 'pro', + duration: usedInput.videoDuration + usedInput.duration, + resolution: '', + fps: 24, + audio: true, + cameraMotion: 'none', + extendVideoPath: copied.path, + extendDuration: usedInput.duration, + extendDirection: usedInput.direction, + }, + takes: [{ + path: copied.path, + bigThumbnailPath: copied.bigThumbnailPath, + smallThumbnailPath: copied.smallThumbnailPath, + width: copied.width, + height: copied.height, + createdAt: Date.now(), + }], + activeTakeIndex: 0, + }) + + extendSubmissionRef.current = null + persistingExtendResultRef.current = null + localStorage.removeItem(GENERATION_RECOVERY_KEY) + setMode('video') + resetExtend() + } catch (error) { + persistingExtendResultRef.current = null + logger.error(`Failed to persist extend result: ${error}`) + setLocalError(createLocalGenerationError('Failed to save extend output to project storage. The app will retry automatically.')) + resetExtend() + } })() }, [extendResult, isExtending, currentProjectId, addAsset, resetExtend]) diff --git a/frontend/views/editor/GapGenerationModal.tsx b/frontend/views/editor/GapGenerationModal.tsx index b63c7c7b5..52fd6d4a5 100644 --- a/frontend/views/editor/GapGenerationModal.tsx +++ b/frontend/views/editor/GapGenerationModal.tsx @@ -17,6 +17,16 @@ interface TimelineGap { type GapGenerateMode = 'text-to-video' | 'image-to-video' | 'text-to-image' +export interface GapFrameConditioning { + imagePath: string | null + imageFile: File | null + reverseResult: boolean +} + +interface FrameOverride { + file: File + previewUrl: string +} interface GapGenerationModalProps { selectedGap: TimelineGap | null @@ -41,7 +51,7 @@ interface GapGenerationModalProps { regenStatusMessage: string regenProgress: number regenReset: () => void - handleGapGenerate: () => void + handleGapGenerate: (conditioning: GapFrameConditioning) => void handleCloseGap: () => void setSelectedGap: (gap: TimelineGap | null) => void gapApplyAudioToTrack: boolean @@ -119,8 +129,8 @@ export function GapGenerationModal({ const [startFrameEnabled, setStartFrameEnabled] = useState(true) const [endFrameEnabled, setEndFrameEnabled] = useState(false) - const [startFrameOverride, setStartFrameOverride] = useState(null) - const [endFrameOverride, setEndFrameOverride] = useState(null) + const [startFrameOverride, setStartFrameOverride] = useState(null) + const [endFrameOverride, setEndFrameOverride] = useState(null) const startFrameInputRef = useRef(null) const endFrameInputRef = useRef(null) @@ -131,22 +141,54 @@ export function GapGenerationModal({ setEndFrameOverride(null) }, [gapGenerateMode]) - const displayedBeforeFrame = startFrameOverride ?? gapBeforeFrame - const displayedAfterFrame = endFrameOverride ?? gapAfterFrame + const displayedBeforeFrame = startFrameOverride?.previewUrl + ?? (gapBeforeFrame ? pathToFileUrl(gapBeforeFrame) : null) + const displayedAfterFrame = endFrameOverride?.previewUrl + ?? (gapAfterFrame ? pathToFileUrl(gapAfterFrame) : null) const handleFrameFileChange = ( e: React.ChangeEvent, - setter: (v: string | null) => void, + setter: (v: FrameOverride | null) => void, onSelect: () => void ) => { const file = e.target.files?.[0] if (!file) return const reader = new FileReader() - reader.onload = (ev) => { setter(ev.target?.result as string); onSelect() } + reader.onload = (ev) => { + setter({ file, previewUrl: ev.target?.result as string }) + onSelect() + } reader.readAsDataURL(file) e.target.value = '' } + const handleGenerate = () => { + if (!isVideoMode || gapImageFile) { + handleGapGenerate({ imagePath: null, imageFile: null, reverseResult: false }) + return + } + + if (startFrameEnabled && (startFrameOverride || gapBeforeFrame)) { + handleGapGenerate({ + imagePath: startFrameOverride ? null : gapBeforeFrame, + imageFile: startFrameOverride?.file ?? null, + reverseResult: false, + }) + return + } + + if (endFrameEnabled && (endFrameOverride || gapAfterFrame)) { + handleGapGenerate({ + imagePath: endFrameOverride ? null : gapAfterFrame, + imageFile: endFrameOverride?.file ?? null, + reverseResult: true, + }) + return + } + + handleGapGenerate({ imagePath: null, imageFile: null, reverseResult: false }) + } + return ( <> {gapGenerateMode && ( @@ -229,7 +271,7 @@ export function GapGenerationModal({ onClick={() => { if (startFrameEnabled) { setStartFrameEnabled(false) } else { setStartFrameEnabled(true); setEndFrameEnabled(false) } }} > { if (endFrameEnabled) { setEndFrameEnabled(false) } else { setEndFrameEnabled(true); setStartFrameEnabled(false) } }} >