dagloop5 commited on
Commit
c70dc2c
·
verified ·
1 Parent(s): 8614379

Delete app.py

Browse files
Files changed (1) hide show
  1. app.py +0 -1079
app.py DELETED
@@ -1,1079 +0,0 @@
1
- import os
2
- import subprocess
3
- import sys
4
-
5
- # Disable torch.compile / dynamo before any torch import
6
- os.environ["TORCH_COMPILE_DISABLE"] = "1"
7
- os.environ["TORCHDYNAMO_DISABLE"] = "1"
8
-
9
- # Clone LTX-2 repo and install packages
10
- LTX_REPO_URL = "https://github.com/Lightricks/LTX-2.git"
11
- LTX_REPO_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "LTX-2")
12
-
13
- LTX_COMMIT = "ae855f8538843825f9015a419cf4ba5edaf5eec2" # known working commit with decode_video
14
-
15
- if not os.path.exists(LTX_REPO_DIR):
16
- print(f"Cloning {LTX_REPO_URL}...")
17
- subprocess.run(["git", "clone", LTX_REPO_URL, LTX_REPO_DIR], check=True)
18
- subprocess.run(["git", "checkout", LTX_COMMIT], cwd=LTX_REPO_DIR, check=True)
19
-
20
- print("Installing ltx-core and ltx-pipelines from cloned repo...")
21
- subprocess.run(
22
- [sys.executable, "-m", "pip", "install", "--force-reinstall", "--no-deps", "-e",
23
- os.path.join(LTX_REPO_DIR, "packages", "ltx-core"),
24
- "-e", os.path.join(LTX_REPO_DIR, "packages", "ltx-pipelines")],
25
- check=True,
26
- )
27
-
28
- sys.path.insert(0, os.path.join(LTX_REPO_DIR, "packages", "ltx-pipelines", "src"))
29
- sys.path.insert(0, os.path.join(LTX_REPO_DIR, "packages", "ltx-core", "src"))
30
-
31
- import logging
32
- import random
33
- import tempfile
34
- from pathlib import Path
35
- import gc
36
- import hashlib
37
-
38
- import torch
39
- torch._dynamo.config.suppress_errors = True
40
- torch._dynamo.config.disable = True
41
-
42
- import spaces
43
- import gradio as gr
44
- import numpy as np
45
- from huggingface_hub import hf_hub_download, snapshot_download
46
- from safetensors.torch import load_file, save_file
47
- from safetensors import safe_open
48
- import json
49
- import requests
50
-
51
- from ltx_core.components.diffusion_steps import Res2sDiffusionStep
52
- from ltx_core.components.guiders import MultiModalGuider, MultiModalGuiderParams
53
- from ltx_core.components.noisers import GaussianNoiser
54
- from ltx_core.components.protocols import DiffusionStepProtocol
55
- from ltx_core.components.schedulers import LTX2Scheduler
56
- from ltx_core.model.audio_vae import encode_audio as vae_encode_audio
57
- from ltx_core.model.audio_vae import decode_audio as vae_decode_audio
58
- from ltx_core.model.upsampler import upsample_video
59
- from ltx_core.model.video_vae import TilingConfig, get_video_chunks_number, decode_video as vae_decode_video
60
- from ltx_core.quantization import QuantizationPolicy
61
- from ltx_core.types import Audio, AudioLatentShape, VideoLatentShape, VideoPixelShape
62
- from ltx_pipelines.distilled import DistilledPipeline
63
- from ltx_pipelines.utils import (
64
- res2s_audio_video_denoising_loop,
65
- cleanup_memory,
66
- encode_prompts,
67
- denoise_audio_video,
68
- )
69
- from ltx_pipelines.utils.args import ImageConditioningInput
70
- from ltx_pipelines.utils.constants import STAGE_2_DISTILLED_SIGMA_VALUES
71
- from ltx_pipelines.utils.helpers import (
72
- combined_image_conditionings,
73
- denoise_video_only,
74
- simple_denoising_func,
75
- multi_modal_guider_denoising_func,
76
- )
77
- from ltx_pipelines.utils.media_io import decode_audio_from_file, encode_video
78
- from ltx_core.loader.primitives import LoraPathStrengthAndSDOps
79
- from ltx_core.loader.sd_ops import LTXV_LORA_COMFY_RENAMING_MAP
80
-
81
- from ltx_core.model.transformer import attention as _attn_mod
82
-
83
- print(f"[ATTN] Before patch: memory_efficient_attention={_attn_mod.memory_efficient_attention}")
84
- try:
85
- from xformers.ops import memory_efficient_attention as _mea
86
- from xformers.ops.fmha import cutlass
87
-
88
- def _cutlass_memory_efficient_attention(*args, **kwargs):
89
- # Force CUTLASS and avoid FlashAttention paths that are crashing.
90
- kwargs["op"] = (cutlass.FwOp, cutlass.BwOp)
91
- return _mea(*args, **kwargs)
92
-
93
- _attn_mod.memory_efficient_attention = _cutlass_memory_efficient_attention
94
- print(f"[ATTN] After patch: memory_efficient_attention={_attn_mod.memory_efficient_attention}")
95
- except Exception as e:
96
- print(f"[ATTN] xformers patch FAILED: {type(e).__name__}: {e}")
97
-
98
- logging.getLogger().setLevel(logging.INFO)
99
-
100
- MAX_SEED = np.iinfo(np.int32).max
101
- DEFAULT_PROMPT = (
102
- "An astronaut hatches from a fragile egg on the surface of the Moon, "
103
- "the shell cracking and peeling apart in gentle low-gravity motion. "
104
- "Fine lunar dust lifts and drifts outward with each movement, floating "
105
- "in slow arcs before settling back onto the ground."
106
- )
107
- DEFAULT_NEGATIVE_PROMPT = (
108
- "worst quality, inconsistent motion, blurry, jittery, distorted, "
109
- "deformed, artifacts, text, watermark, logo, frame, border, "
110
- "low resolution, pixelated, unnatural, fake, CGI, cartoon"
111
- )
112
- DEFAULT_FRAME_RATE = 24.0
113
-
114
- # Resolution presets: (width, height)
115
- RESOLUTIONS = {
116
- "high": {"16:9": (1536, 1024), "9:16": (1024, 1536), "1:1": (1024, 1024)},
117
- "low": {"16:9": (768, 512), "9:16": (512, 768), "1:1": (768, 768)},
118
- }
119
-
120
- class LTX23DistilledA2VPipeline:
121
- """Standalone pipeline with optional audio conditioning — no parent class."""
122
-
123
- def __init__(
124
- self,
125
- distilled_checkpoint_path: str,
126
- istilled_lora_path: str,
127
- distilled_lora_strength_stage_1: float,
128
- distilled_lora_strength_stage_2: float,
129
- spatial_upsampler_path: str,
130
- gemma_root: str,
131
- loras: tuple,
132
- quantization: QuantizationPolicy | None = None,
133
- ):
134
- from ltx_pipelines.utils import ModelLedger, denoise_audio_video
135
- from ltx_pipelines.utils.types import PipelineComponents
136
-
137
- self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
138
- self.dtype = torch.bfloat16
139
-
140
- distilled_lora_stage_1 = LoraPathStrengthAndSDOps(
141
- path=distilled_lora_path,
142
- strength=distilled_lora_strength_stage_1,
143
- sd_ops=LTXV_LORA_COMFY_RENAMING_MAP,
144
- )
145
- distilled_lora_stage_2 = LoraPathStrengthAndSDOps(
146
- path=distilled_lora_path,
147
- strength=distilled_lora_strength_stage_2,
148
- sd_ops=LTXV_LORA_COMFY_RENAMING_MAP,
149
- )
150
-
151
- self.stage_1_model_ledger = ModelLedger(
152
- dtype=self.dtype,
153
- device=device,
154
- checkpoint_path=checkpoint_path,
155
- gemma_root_path=gemma_root,
156
- spatial_upsampler_path=spatial_upsampler_path,
157
- loras=(*loras, distilled_lora_stage_1),
158
- quantization=quantization,
159
- )
160
-
161
- self.stage_2_model_ledger = self.stage_1_model_ledger.with_loras(
162
- loras=(*loras, distilled_lora_stage_2),
163
- )
164
-
165
- self.pipeline_components = PipelineComponents(
166
- dtype=self.dtype,
167
- device=self.device,
168
- )
169
-
170
- def __call__(
171
- self,
172
- prompt: str,
173
- negative_prompt: str,
174
- seed: int,
175
- height: int,
176
- width: int,
177
- num_frames: int,
178
- frame_rate: float,
179
- video_guider_params: MultiModalGuiderParams,
180
- audio_guider_params: MultiModalGuiderParams,
181
- images: list[ImageConditioningInput],
182
- num_inference_steps: int = 8,
183
- audio_path: str | None = None,
184
- tiling_config: TilingConfig | None = None,
185
- enhance_prompt: bool = False,
186
- ):
187
- print(prompt)
188
-
189
- generator = torch.Generator(device=self.device).manual_seed(seed)
190
- noiser = GaussianNoiser(generator=generator)
191
- stepper = Res2sDiffusionStep()
192
- dtype = torch.bfloat16
193
-
194
- ctx_p, ctx_n = encode_prompts(
195
- [prompt, negative_prompt],
196
- self.model_ledger,
197
- enhance_first_prompt=enhance_prompt,
198
- enhance_prompt_image=images[0].path if len(images) > 0 else None,
199
- )
200
- v_context_p, a_context_p = ctx_p.video_encoding, ctx_p.audio_encoding
201
- v_context_n, a_context_n = ctx_n.video_encoding, ctx_n.audio_encoding
202
-
203
- # ── Audio encoding (only for conditioning, not output generation) ──
204
- encoded_audio_latent = None
205
- decoded_audio = None
206
- if audio_path is not None:
207
- video_duration = num_frames / frame_rate
208
- decoded_audio = decode_audio_from_file(audio_path, self.device, 0.0, video_duration)
209
- if decoded_audio is None:
210
- raise ValueError(f"Could not extract audio stream from {audio_path}")
211
-
212
- encoded_audio_latent = vae_encode_audio(decoded_audio, self.model_ledger.audio_encoder())
213
- audio_shape = AudioLatentShape.from_duration(batch=1, duration=video_duration, channels=8, mel_bins=16)
214
- expected_frames = audio_shape.frames
215
- actual_frames = encoded_audio_latent.shape[2]
216
-
217
- if actual_frames > expected_frames:
218
- encoded_audio_latent = encoded_audio_latent[:, :, :expected_frames, :]
219
- elif actual_frames < expected_frames:
220
- pad = torch.zeros(
221
- encoded_audio_latent.shape[0],
222
- encoded_audio_latent.shape[1],
223
- expected_frames - actual_frames,
224
- encoded_audio_latent.shape[3],
225
- device=encoded_audio_latent.device,
226
- dtype=encoded_audio_latent.dtype,
227
- )
228
- encoded_audio_latent = torch.cat([encoded_audio_latent, pad], dim=2)
229
-
230
- video_encoder = self.stage_1_model_ledger.video_encoder()
231
- transformer = self.stage_1_model_ledger.transformer()
232
-
233
- # Stage 1: Generate sigmas using LTX2Scheduler with user-specified steps
234
- empty_latent = torch.empty(VideoLatentShape.from_pixel_shape(
235
- VideoPixelShape(batch=1, frames=num_frames, width=width // 2, height=height // 2, fps=frame_rate)
236
- ).to_torch_shape())
237
- stage_1_sigmas = (
238
- LTX2Scheduler()
239
- .execute(latent=empty_latent, steps=num_inference_steps)
240
- .to(dtype=torch.float32, device=self.device)
241
- )
242
-
243
- def stage1_denoising_loop(sigmas: torch.Tensor, video_state, audio_state, stepper: DiffusionStepProtocol):
244
- return res2s_audio_video_denoising_loop(
245
- sigmas=sigmas,
246
- video_state=video_state,
247
- audio_state=audio_state,
248
- stepper=stepper,
249
- denoise_fn=multi_modal_guider_denoising_func(
250
- video_guider=MultiModalGuider(
251
- params=video_guider_params,
252
- negative_context=v_context_n,
253
- ),
254
- audio_guider=MultiModalGuider(
255
- params=audio_guider_params,
256
- negative_context=a_context_n,
257
- ),
258
- v_context=v_context_p,
259
- a_context=a_context_p,
260
- transformer=transformer,
261
- ),
262
- )
263
-
264
- def stage2_denoising_loop(sigmas: torch.Tensor, video_state, audio_state, stepper: DiffusionStepProtocol):
265
- return res2s_audio_video_denoising_loop(
266
- sigmas=sigmas,
267
- video_state=video_state,
268
- audio_state=audio_state,
269
- stepper=stepper,
270
- denoise_fn=simple_denoising_func(
271
- video_context=v_context_p,
272
- audio_context=a_context_p,
273
- transformer=transformer, # noqa: F821
274
- ),
275
- )
276
-
277
- # ── Stage 1: Half resolution ──
278
- stage_1_output_shape = VideoPixelShape(
279
- batch=1,
280
- frames=num_frames,
281
- width=width // 2,
282
- height=height // 2,
283
- fps=frame_rate,
284
- )
285
- stage_1_conditionings = combined_image_conditionings(
286
- images=images,
287
- height=stage_1_output_shape.height,
288
- width=stage_1_output_shape.width,
289
- video_encoder=video_encoder,
290
- dtype=dtype,
291
- device=self.device,
292
- )
293
-
294
- # Use denoise_audio_video so audio is ALWAYS generated
295
- from ltx_pipelines.utils import denoise_audio_video
296
- video_state, audio_state = denoise_audio_video(
297
- output_shape=stage_1_output_shape,
298
- conditionings=stage_1_conditionings,
299
- noiser=noiser,
300
- sigmas=stage_1_sigmas,
301
- stepper=stepper,
302
- denoising_loop_fn=stage1_denoising_loop,
303
- components=self.pipeline_components,
304
- dtype=dtype,
305
- device=self.device,
306
- initial_audio_latent=encoded_audio_latent,
307
- )
308
-
309
- torch.cuda.synchronize()
310
- cleanup_memory()
311
-
312
- # ── Upscaling ──
313
- video_encoder = self.stage_1_model_ledger.video_encoder()
314
- upscaled_video_latent = upsample_video(
315
- latent=video_state.latent[:1],
316
- video_encoder=video_encoder,
317
- upsampler=self.stage_2_model_ledger.spatial_upsampler(),
318
- )
319
-
320
- # ── Stage 2: Full resolution ──
321
- stage_2_sigmas = torch.tensor(STAGE_2_DISTILLED_SIGMA_VALUES, device=self.device)
322
- stage_2_output_shape = VideoPixelShape(batch=1, frames=num_frames, width=width, height=height, fps=frame_rate)
323
- stage_2_conditionings = combined_image_conditionings(
324
- images=images,
325
- height=stage_2_output_shape.height,
326
- width=stage_2_output_shape.width,
327
- video_encoder=video_encoder,
328
- dtype=dtype,
329
- device=self.device,
330
- )
331
-
332
- transformer = self.stage_2_model_ledger.transformer()
333
-
334
- video_state, audio_state = denoise_audio_video(
335
- output_shape=stage_2_output_shape,
336
- conditionings=stage_2_conditionings,
337
- noiser=noiser,
338
- sigmas=stage_2_sigmas,
339
- stepper=stepper,
340
- denoising_loop_fn=stage2_denoising_loop,
341
- components=self.pipeline_components,
342
- dtype=dtype,
343
- device=self.device,
344
- noise_scale=stage_2_sigmas[0],
345
- initial_video_latent=upscaled_video_latent,
346
- initial_audio_latent=audio_state.latent,
347
- )
348
-
349
- torch.cuda.synchronize()
350
- cleanup_memory()
351
-
352
- # ── Decode both video and audio ──
353
- decoded_video = vae_decode_video(
354
- video_state.latent,
355
- self.stage_2_model_ledger.video_decoder(),
356
- tiling_config,
357
- generator,
358
- )
359
- decoded_audio_output = vae_decode_audio(
360
- audio_state.latent,
361
- self.stage_2_model_ledger.audio_decoder(),
362
- self.stage_2_model_ledger.vocoder(),
363
- )
364
-
365
- return decoded_video, decoded_audio_output
366
-
367
- # Model repos
368
- LTX_MODEL_REPO = "TenStrip/LTX2.3-10Eros"
369
- GEMMA_REPO ="Lightricks/gemma-3-12b-it-qat-q4_0-unquantized"
370
- DISTILLED_LORA_REPO = "TenStrip/LTX2.3_Distilled_Lora_1.1_Experiments"
371
-
372
- # Download model checkpoints
373
- print("=" * 80)
374
- print("Downloading LTX-2.3 distilled model + Gemma...")
375
- print("=" * 80)
376
-
377
- # LoRA cache directory and currently-applied key
378
- LORA_CACHE_DIR = Path("lora_cache")
379
- LORA_CACHE_DIR.mkdir(exist_ok=True)
380
- current_lora_key: str | None = None
381
-
382
- PENDING_LORA_KEY: str | None = None
383
- PENDING_LORA_STATE: dict[str, torch.Tensor] | None = None
384
- PENDING_LORA_STATUS: str = "No LoRA state prepared yet."
385
-
386
- weights_dir = Path("weights")
387
- weights_dir.mkdir(exist_ok=True)
388
- checkpoint_path = hf_hub_download(
389
- repo_id=LTX_MODEL_REPO,
390
- filename="10Eros_v1_bf16.safetensors",
391
- local_dir=str(weights_dir),
392
- local_dir_use_symlinks=False,
393
- )
394
- distilled_lora_path = hf_hub_download(repo_id=DISTILLED_LORA_REPO, filename="ltx-2.3-22b-distilled-lora-1.1_fro90_ceil72_condsafe.safetensors")
395
- spatial_upsampler_path = hf_hub_download(repo_id="Lightricks/LTX-2.3", filename="ltx-2.3-spatial-upscaler-x2-1.1.safetensors")
396
- gemma_root = snapshot_download(repo_id=GEMMA_REPO)
397
-
398
-
399
- LORA_REPO = "dagloop5/LoRA"
400
-
401
- print("=" * 80)
402
- print("Downloading LoRA adapters from dagloop5/LoRA...")
403
- print("=" * 80)
404
- pose_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="LTX2_3_NSFW_furry_concat_v2.safetensors")
405
- general_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="LTX2.3_reasoning_I2V_V3.safetensors")
406
- motion_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="motion_helper.safetensors")
407
- dreamlay_lora_path = hf_hub_download(repo_id="lynaNSFW/DR34ML4Y_AIO_NSFW_LTX23", filename="DR34ML4Y_LTXXX_V2.safetensors") # m15510n4ry, bl0wj0b, d0ubl3_bj, d0gg1e, c0wg1rl
408
- mself_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="Furry Hyper Masturbation - LTX-2 I2V v1.safetensors") # Hyperfap
409
- dramatic_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="LTX-2.3 - Orgasm.safetensors") # "[He | She] is having am orgasm." (am or an?)
410
- fluid_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="Cr3ampi3_animation_sulphur-2_i2v_v1.0.safetensors") # cr3ampi3 animation
411
- liquid_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="liquid_wet_dr1pp_ltx2_v1.0_scaled.safetensors") # wet dr1pp
412
- demopose_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="clapping-cheeks-audio-v001-alpha.safetensors")
413
- voice_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="hentai_voice_ltx23.safetensors")
414
- realism_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="FurryenhancerLTX2.3V4.094fused.safetensors")
415
- transition_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="LTX-2_takerpov_lora_v1.2.safetensors") # takerpov1, taker pov
416
- physics_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="LTX2.3_Physics_V2_000002000.safetensors")
417
- reasoning_lora_path = hf_hub_download(repo_id="LiconStudio/Ltx2.3-VBVR-lora-I2V", filename="Ltx2.3-Licon-VBVR-I2V-390K-R32.safetensors")
418
- twostep_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="LTX2.3_Multi_step_video_reasoning_V0.1.safetensors")
419
-
420
- print(f"Pose LoRA: {pose_lora_path}")
421
- print(f"General LoRA: {general_lora_path}")
422
- print(f"Motion LoRA: {motion_lora_path}")
423
- print(f"Dreamlay LoRA: {dreamlay_lora_path}")
424
- print(f"Mself LoRA: {mself_lora_path}")
425
- print(f"Dramatic LoRA: {dramatic_lora_path}")
426
- print(f"Fluid LoRA: {fluid_lora_path}")
427
- print(f"Liquid LoRA: {liquid_lora_path}")
428
- print(f"Demopose LoRA: {demopose_lora_path}")
429
- print(f"Voice LoRA: {voice_lora_path}")
430
- print(f"Realism LoRA: {realism_lora_path}")
431
- print(f"Transition LoRA: {transition_lora_path}")
432
- print(f"Physics LoRA: {physics_lora_path}")
433
- print(f"Reasoning LoRA: {reasoning_lora_path}")
434
- print(f"Twostep LoRA: {twostep_lora_path}")
435
-
436
- print(f"Checkpoint: {checkpoint_path}")
437
- print(f"Spatial upsampler: {spatial_upsampler_path}")
438
- print(f"[Gemma] Root ready: {gemma_root}")
439
-
440
- pipeline = LTX23DistilledA2VPipeline(
441
- distilled_checkpoint_path=checkpoint_path,
442
- distilled_lora=distilled_lora_path,
443
- distilled_lora_strength_stage_1=1.0,
444
- distilled_lora_strength_stage_2=0.5,
445
- spatial_upsampler_path=spatial_upsampler_path,
446
- gemma_root=gemma_root,
447
- loras=[],
448
- quantization=QuantizationPolicy.fp8_cast(),
449
- )
450
-
451
- def _make_lora_key(pose_strength: float, general_strength: float, motion_strength: float, dreamlay_strength: float, mself_strength: float, dramatic_strength: float, fluid_strength: float, liquid_strength: float, demopose_strength: float, voice_strength: float, realism_strength: float, transition_strength: float, physics_strength: float, reasoning_strength: float, twostep_strength: float) -> tuple[str, str]:
452
- rp = round(float(pose_strength), 2)
453
- rg = round(float(general_strength), 2)
454
- rm = round(float(motion_strength), 2)
455
- rd = round(float(dreamlay_strength), 2)
456
- rs = round(float(mself_strength), 2)
457
- rr = round(float(dramatic_strength), 2)
458
- rf = round(float(fluid_strength), 2)
459
- rl = round(float(liquid_strength), 2)
460
- ro = round(float(demopose_strength), 2)
461
- rv = round(float(voice_strength), 2)
462
- re = round(float(realism_strength), 2)
463
- rt = round(float(transition_strength), 2)
464
- ry = round(float(physics_strength), 2)
465
- ri = round(float(reasoning_strength), 2)
466
- rw = round(float(twostep_strength), 2)
467
- key_str = f"{pose_lora_path}:{rp}|{general_lora_path}:{rg}|{motion_lora_path}:{rm}|{dreamlay_lora_path}:{rd}|{mself_lora_path}:{rs}|{dramatic_lora_path}:{rr}|{fluid_lora_path}:{rf}|{liquid_lora_path}:{rl}|{demopose_lora_path}:{ro}|{voice_lora_path}:{rv}|{realism_lora_path}:{re}|{transition_lora_path}:{rt}|{physics_lora_path}:{ry}|{reasoning_lora_path}:{ri}|{twostep_lora_path}:{rw}"
468
- key = hashlib.sha256(key_str.encode("utf-8")).hexdigest()
469
- return key, key_str
470
-
471
-
472
- def prepare_lora_cache(
473
- pose_strength: float,
474
- general_strength: float,
475
- motion_strength: float,
476
- dreamlay_strength: float,
477
- mself_strength: float,
478
- dramatic_strength: float,
479
- fluid_strength: float,
480
- liquid_strength: float,
481
- demopose_strength: float,
482
- voice_strength: float,
483
- realism_strength: float,
484
- transition_strength: float,
485
- physics_strength: float,
486
- reasoning_strength: float,
487
- twostep_strength: float,
488
- progress=gr.Progress(track_tqdm=True),
489
- ):
490
- """
491
- CPU-only step:
492
- - checks cache
493
- - loads cached fused transformer state_dict, or
494
- - builds fused transformer on CPU and saves it
495
- The resulting state_dict is stored in memory and can be applied later.
496
- """
497
- global PENDING_LORA_KEY, PENDING_LORA_STATE, PENDING_LORA_STATUS
498
-
499
- ledger = pipeline.model_ledger
500
- key, _ = _make_lora_key(pose_strength, general_strength, motion_strength, dreamlay_strength, mself_strength, dramatic_strength, fluid_strength, liquid_strength, demopose_strength, voice_strength, realism_strength, transition_strength, physics_strength, reasoning_strength, twostep_strength)
501
- cache_path = LORA_CACHE_DIR / f"{key}.safetensors"
502
-
503
- progress(0.05, desc="Preparing LoRA state")
504
- if cache_path.exists():
505
- try:
506
- progress(0.20, desc="Loading cached fused state")
507
- state = load_file(str(cache_path))
508
- PENDING_LORA_KEY = key
509
- PENDING_LORA_STATE = state
510
- PENDING_LORA_STATUS = f"Loaded cached LoRA state: {cache_path.name}"
511
- return PENDING_LORA_STATUS
512
- except Exception as e:
513
- print(f"[LoRA] Cache load failed: {type(e).__name__}: {e}")
514
-
515
- entries = [
516
- (pose_lora_path, round(float(pose_strength), 2)),
517
- (general_lora_path, round(float(general_strength), 2)),
518
- (motion_lora_path, round(float(motion_strength), 2)),
519
- (dreamlay_lora_path, round(float(dreamlay_strength), 2)),
520
- (mself_lora_path, round(float(mself_strength), 2)),
521
- (dramatic_lora_path, round(float(dramatic_strength), 2)),
522
- (fluid_lora_path, round(float(fluid_strength), 2)),
523
- (liquid_lora_path, round(float(liquid_strength), 2)),
524
- (demopose_lora_path, round(float(demopose_strength), 2)),
525
- (voice_lora_path, round(float(voice_strength), 2)),
526
- (realism_lora_path, round(float(realism_strength), 2)),
527
- (transition_lora_path, round(float(transition_strength), 2)),
528
- (physics_lora_path, round(float(physics_strength), 2)),
529
- (reasoning_lora_path, round(float(reasoning_strength), 2)),
530
- (twostep_lora_path, round(float(twostep_strength), 2)),
531
- ]
532
- loras_for_builder = [
533
- LoraPathStrengthAndSDOps(path, strength, LTXV_LORA_COMFY_RENAMING_MAP)
534
- for path, strength in entries
535
- if path is not None and float(strength) != 0.0
536
- ]
537
-
538
- if not loras_for_builder:
539
- PENDING_LORA_KEY = None
540
- PENDING_LORA_STATE = None
541
- PENDING_LORA_STATUS = "No non-zero LoRA strengths selected; nothing to prepare."
542
- return PENDING_LORA_STATUS
543
-
544
- tmp_ledger = None
545
- new_transformer_cpu = None
546
- try:
547
- progress(0.35, desc="Building fused CPU transformer")
548
- tmp_ledger = pipeline.model_ledger.__class__(
549
- dtype=ledger.dtype,
550
- device=torch.device("cpu"),
551
- checkpoint_path=str(checkpoint_path),
552
- spatial_upsampler_path=str(spatial_upsampler_path),
553
- gemma_root_path=str(gemma_root),
554
- loras=tuple(loras_for_builder),
555
- quantization=getattr(ledger, "quantization", None),
556
- )
557
- new_transformer_cpu = tmp_ledger.transformer()
558
-
559
- progress(0.70, desc="Extracting fused state_dict")
560
- state = {
561
- k: v.detach().cpu().contiguous()
562
- for k, v in new_transformer_cpu.state_dict().items()
563
- }
564
- save_file(state, str(cache_path))
565
-
566
- PENDING_LORA_KEY = key
567
- PENDING_LORA_STATE = state
568
- PENDING_LORA_STATUS = f"Built and cached LoRA state: {cache_path.name}"
569
- return PENDING_LORA_STATUS
570
-
571
- except Exception as e:
572
- import traceback
573
- print(f"[LoRA] Prepare failed: {type(e).__name__}: {e}")
574
- print(traceback.format_exc())
575
- PENDING_LORA_KEY = None
576
- PENDING_LORA_STATE = None
577
- PENDING_LORA_STATUS = f"LoRA prepare failed: {type(e).__name__}: {e}"
578
- return PENDING_LORA_STATUS
579
-
580
- finally:
581
- try:
582
- del new_transformer_cpu
583
- except Exception:
584
- pass
585
- try:
586
- del tmp_ledger
587
- except Exception:
588
- pass
589
- gc.collect()
590
-
591
-
592
- def apply_prepared_lora_state_to_pipeline():
593
- """
594
- Fast step: copy the already prepared CPU state into the live transformer.
595
- This is the only part that should remain near generation time.
596
- """
597
- global current_lora_key, PENDING_LORA_KEY, PENDING_LORA_STATE
598
-
599
- if PENDING_LORA_STATE is None or PENDING_LORA_KEY is None:
600
- print("[LoRA] No prepared LoRA state available; skipping.")
601
- return False
602
-
603
- if current_lora_key == PENDING_LORA_KEY:
604
- print("[LoRA] Prepared LoRA state already active; skipping.")
605
- return True
606
-
607
- existing_transformer = _transformer
608
- with torch.no_grad():
609
- missing, unexpected = existing_transformer.load_state_dict(PENDING_LORA_STATE, strict=False)
610
- if missing or unexpected:
611
- print(f"[LoRA] load_state_dict mismatch: missing={len(missing)}, unexpected={len(unexpected)}")
612
-
613
- current_lora_key = PENDING_LORA_KEY
614
- print("[LoRA] Prepared LoRA state applied to the pipeline.")
615
- return True
616
-
617
- # Preload all models for ZeroGPU tensor packing.
618
- print("Preloading all models (including Gemma and audio components)...")
619
-
620
- # We now have two ledgers — stage 1 (distilled LoRA @ 1.0) and stage 2 (distilled LoRA @ 0.5).
621
- # Both share the same dev checkpoint and spatial upsampler; only the transformer differs.
622
- ledger_s1 = pipeline.stage_1_model_ledger
623
- ledger_s2 = pipeline.stage_2_model_ledger
624
-
625
- # Save the original factory methods from BOTH ledgers so we can rebuild individual components later.
626
- _orig_transformer_factory_s1 = ledger_s1.transformer
627
- _orig_transformer_factory_s2 = ledger_s2.transformer
628
- _orig_video_encoder_factory = ledger_s1.video_encoder
629
- _orig_video_decoder_factory = ledger_s1.video_decoder
630
- _orig_audio_encoder_factory = ledger_s1.audio_encoder
631
- _orig_audio_decoder_factory = ledger_s1.audio_decoder
632
- _orig_vocoder_factory = ledger_s1.vocoder
633
- _orig_spatial_upsampler_factory = ledger_s1.spatial_upsampler
634
- _orig_text_encoder_factory = ledger_s1.text_encoder
635
- _orig_gemma_embeddings_factory = ledger_s1.gemma_embeddings_processor
636
-
637
- # Call the factories to create cached instances.
638
- # Stage 1 transformer: dev checkpoint + distilled LoRA @ strength 1.0 (baked in at build time)
639
- _transformer_s1 = _orig_transformer_factory_s1()
640
- # Stage 2 transformer: dev checkpoint + distilled LoRA @ strength 0.5 (baked in at build time)
641
- _transformer_s2 = _orig_transformer_factory_s2()
642
-
643
- # Shared components — only need one copy since both ledgers use the same VAE/encoder paths.
644
- _video_encoder = _orig_video_encoder_factory()
645
- _video_decoder = _orig_video_decoder_factory()
646
- _audio_encoder = _orig_audio_encoder_factory()
647
- _audio_decoder = _orig_audio_decoder_factory()
648
- _vocoder = _orig_vocoder_factory()
649
- _spatial_upsampler = _orig_spatial_upsampler_factory()
650
- _text_encoder = _orig_text_encoder_factory()
651
- _embeddings_processor = _orig_gemma_embeddings_factory()
652
-
653
- # Replace ledger methods with lightweight lambdas that return the cached instances.
654
- # Both ledgers point to the same shared model instances (except transformer).
655
- # We keep the original factories above so we can call them later to rebuild components.
656
- ledger_s1.transformer = lambda: _transformer_s1
657
- ledger_s2.transformer = lambda: _transformer_s2
658
- ledger_s1.video_encoder = lambda: _video_encoder
659
- ledger_s2.video_encoder = lambda: _video_encoder
660
- ledger_s1.video_decoder = lambda: _video_decoder
661
- ledger_s2.video_decoder = lambda: _video_decoder
662
- ledger_s1.audio_encoder = lambda: _audio_encoder
663
- ledger_s2.audio_encoder = lambda: _audio_encoder
664
- ledger_s1.audio_decoder = lambda: _audio_decoder
665
- ledger_s2.audio_decoder = lambda: _audio_decoder
666
- ledger_s1.vocoder = lambda: _vocoder
667
- ledger_s2.vocoder = lambda: _vocoder
668
- ledger_s1.spatial_upsampler = lambda: _spatial_upsampler
669
- ledger_s2.spatial_upsampler = lambda: _spatial_upsampler
670
- ledger_s1.text_encoder = lambda: _text_encoder
671
- ledger_s2.text_encoder = lambda: _text_encoder
672
- ledger_s1.gemma_embeddings_processor = lambda: _embeddings_processor
673
- ledger_s2.gemma_embeddings_processor = lambda: _embeddings_processor
674
-
675
- print("All models preloaded (including Gemma text encoder and audio encoder)!")
676
- print(f" Stage 1 transformer: {_transformer_s1.__class__.__name__} (distilled LoRA @ 1.0 baked)")
677
- print(f" Stage 2 transformer: {_transformer_s2.__class__.__name__} (distilled LoRA @ 0.5 baked)")
678
- # ---- REPLACE PRELOAD BLOCK END ----
679
-
680
- print("=" * 80)
681
- print("Pipeline ready!")
682
- print("=" * 80)
683
-
684
- def log_memory(tag: str):
685
- if torch.cuda.is_available():
686
- allocated = torch.cuda.memory_allocated() / 1024**3
687
- peak = torch.cuda.max_memory_allocated() / 1024**3
688
- free, total = torch.cuda.mem_get_info()
689
- print(f"[VRAM {tag}] allocated={allocated:.2f}GB peak={peak:.2f}GB free={free / 1024**3:.2f}GB total={total / 1024**3:.2f}GB")
690
-
691
-
692
- def detect_aspect_ratio(image) -> str:
693
- if image is None:
694
- return "16:9"
695
- if hasattr(image, "size"):
696
- w, h = image.size
697
- elif hasattr(image, "shape"):
698
- h, w = image.shape[:2]
699
- else:
700
- return "16:9"
701
- ratio = w / h
702
- candidates = {"16:9": 16 / 9, "9:16": 9 / 16, "1:1": 1.0}
703
- return min(candidates, key=lambda k: abs(ratio - candidates[k]))
704
-
705
-
706
- def on_image_upload(first_image, last_image, high_res):
707
- ref_image = first_image if first_image is not None else last_image
708
- aspect = detect_aspect_ratio(ref_image)
709
- tier = "high" if high_res else "low"
710
- w, h = RESOLUTIONS[tier][aspect]
711
- return gr.update(value=w), gr.update(value=h)
712
-
713
-
714
- def on_highres_toggle(first_image, last_image, high_res):
715
- ref_image = first_image if first_image is not None else last_image
716
- aspect = detect_aspect_ratio(ref_image)
717
- tier = "high" if high_res else "low"
718
- w, h = RESOLUTIONS[tier][aspect]
719
- return gr.update(value=w), gr.update(value=h)
720
-
721
- def get_gpu_duration(
722
- first_image,
723
- last_image,
724
- input_audio,
725
- prompt: str,
726
- negative_prompt: str,
727
- duration: float,
728
- gpu_duration: float,
729
- enhance_prompt: bool = True,
730
- seed: int = 42,
731
- randomize_seed: bool = True,
732
- height: int = 1024,
733
- width: int = 1536,
734
- video_cfg_scale: float = 1.0,
735
- video_stg_scale: float = 0.0,
736
- video_rescale_scale: float = 0.45,
737
- video_a2v_scale: float = 3.0,
738
- audio_cfg_scale: float = 1.0,
739
- audio_stg_scale: float = 0.0,
740
- audio_rescale_scale: float = 1.0,
741
- audio_v2a_scale: float = 3.0,
742
- pose_strength: float = 0.0,
743
- general_strength: float = 0.0,
744
- motion_strength: float = 0.0,
745
- dreamlay_strength: float = 0.0,
746
- mself_strength: float = 0.0,
747
- dramatic_strength: float = 0.0,
748
- fluid_strength: float = 0.0,
749
- liquid_strength: float = 0.0,
750
- demopose_strength: float = 0.0,
751
- voice_strength: float = 0.0,
752
- realism_strength: float = 0.0,
753
- transition_strength: float = 0.0,
754
- physics_strength: float = 0.0,
755
- reasoning_strength: float = 0.0,
756
- twostep_strength: float = 0.0,
757
- num_inference_steps: int = 8,
758
- progress=None,
759
- ):
760
- return int(gpu_duration)
761
-
762
- @spaces.GPU(duration=get_gpu_duration)
763
- @torch.inference_mode()
764
- def generate_video(
765
- first_image,
766
- last_image,
767
- input_audio,
768
- prompt: str,
769
- negative_prompt: str,
770
- duration: float,
771
- gpu_duration: float,
772
- enhance_prompt: bool = True,
773
- seed: int = 42,
774
- randomize_seed: bool = True,
775
- height: int = 1024,
776
- width: int = 1536,
777
- video_cfg_scale: float = 1.0,
778
- video_stg_scale: float = 0.0,
779
- video_rescale_scale: float = 0.45,
780
- video_a2v_scale: float = 3.0,
781
- audio_cfg_scale: float = 1.0,
782
- audio_stg_scale: float = 0.0,
783
- audio_rescale_scale: float = 1.0,
784
- audio_v2a_scale: float = 3.0,
785
- pose_strength: float = 0.0,
786
- general_strength: float = 0.0,
787
- motion_strength: float = 0.0,
788
- dreamlay_strength: float = 0.0,
789
- mself_strength: float = 0.0,
790
- dramatic_strength: float = 0.0,
791
- fluid_strength: float = 0.0,
792
- liquid_strength: float = 0.0,
793
- demopose_strength: float = 0.0,
794
- voice_strength: float = 0.0,
795
- realism_strength: float = 0.0,
796
- transition_strength: float = 0.0,
797
- physics_strength: float = 0.0,
798
- reasoning_strength: float = 0.0,
799
- twostep_strength: float = 0.0,
800
- num_inference_steps: int = 8,
801
- progress=gr.Progress(track_tqdm=True),
802
- ):
803
- try:
804
- torch.cuda.reset_peak_memory_stats()
805
- log_memory("start")
806
-
807
- current_seed = random.randint(0, MAX_SEED) if randomize_seed else int(seed)
808
-
809
- frame_rate = DEFAULT_FRAME_RATE
810
- num_frames = int(duration * frame_rate) + 1
811
- num_frames = ((num_frames - 1 + 7) // 8) * 8 + 1
812
-
813
- print(f"Generating: {height}x{width}, {num_frames} frames ({duration}s), seed={current_seed}")
814
-
815
- images = []
816
- output_dir = Path("outputs")
817
- output_dir.mkdir(exist_ok=True)
818
-
819
- if first_image is not None:
820
- temp_first_path = output_dir / f"temp_first_{current_seed}.jpg"
821
- if hasattr(first_image, "save"):
822
- first_image.save(temp_first_path)
823
- else:
824
- temp_first_path = Path(first_image)
825
- images.append(ImageConditioningInput(path=str(temp_first_path), frame_idx=0, strength=1.0))
826
-
827
- if last_image is not None:
828
- temp_last_path = output_dir / f"temp_last_{current_seed}.jpg"
829
- if hasattr(last_image, "save"):
830
- last_image.save(temp_last_path)
831
- else:
832
- temp_last_path = Path(last_image)
833
- images.append(ImageConditioningInput(path=str(temp_last_path), frame_idx=num_frames - 1, strength=1.0))
834
-
835
- tiling_config = TilingConfig.default()
836
- video_chunks_number = get_video_chunks_number(num_frames, tiling_config)
837
-
838
- video_guider_params = MultiModalGuiderParams(
839
- cfg_scale=video_cfg_scale,
840
- stg_scale=video_stg_scale,
841
- rescale_scale=video_rescale_scale,
842
- modality_scale=video_a2v_scale,
843
- skip_step=0,
844
- stg_blocks=[],
845
- )
846
-
847
- audio_guider_params = MultiModalGuiderParams(
848
- cfg_scale=audio_cfg_scale,
849
- stg_scale=audio_stg_scale,
850
- rescale_scale=audio_rescale_scale,
851
- modality_scale=audio_v2a_scale,
852
- skip_step=0,
853
- stg_blocks=[],
854
- )
855
-
856
- log_memory("before pipeline call")
857
-
858
- apply_prepared_lora_state_to_pipeline()
859
-
860
- video, audio = pipeline(
861
- prompt=prompt,
862
- negative_prompt=negative_prompt,
863
- seed=current_seed,
864
- height=int(height),
865
- width=int(width),
866
- num_frames=num_frames,
867
- frame_rate=frame_rate,
868
- video_guider_params=video_guider_params,
869
- audio_guider_params=audio_guider_params,
870
- images=images,
871
- num_inference_steps=num_inference_steps,
872
- audio_path=input_audio,
873
- tiling_config=tiling_config,
874
- enhance_prompt=enhance_prompt,
875
- )
876
-
877
- log_memory("after pipeline call")
878
-
879
- output_path = tempfile.mktemp(suffix=".mp4")
880
- encode_video(
881
- video=video,
882
- fps=frame_rate,
883
- audio=audio,
884
- output_path=output_path,
885
- video_chunks_number=video_chunks_number,
886
- )
887
-
888
- log_memory("after encode_video")
889
- return str(output_path), current_seed
890
-
891
- except Exception as e:
892
- import traceback
893
- log_memory("on error")
894
- print(f"Error: {str(e)}\n{traceback.format_exc()}")
895
- return None, current_seed
896
-
897
- # =============================================================================
898
- # Gradio UI
899
- # =============================================================================
900
-
901
- css = """
902
- .fillable {max-width: 1200px !important}
903
- .progress-text {color: black}
904
- """
905
-
906
- with gr.Blocks(title="LTX-2.3 Distilled with LoRAs, Negative Prompting, and Advanced Settings") as demo:
907
- gr.Markdown("# LTX-2.3 Two-Stage HQ Video Generation")
908
- gr.Markdown(
909
- "High-quality text/image-to-video with cached LoRA state + CFG guidance. "
910
- "[[Model]](https://huggingface.co/Lightricks/LTX-2.3)"
911
- )
912
-
913
- with gr.Row():
914
- # LEFT SIDE: Input Controls
915
- with gr.Column():
916
- with gr.Row():
917
- first_image = gr.Image(label="First Frame (Optional)", type="pil")
918
- last_image = gr.Image(label="Last Frame (Optional)", type="pil")
919
-
920
- prompt = gr.Textbox(
921
- label="Prompt",
922
- value="Make this image come alive with cinematic motion, smooth animation",
923
- lines=3,
924
- placeholder="Describe the motion and animation you want...",
925
- )
926
-
927
- negative_prompt = gr.Textbox(
928
- label="Negative Prompt",
929
- value="blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts.",
930
- lines=2,
931
- )
932
-
933
- duration = gr.Slider(
934
- label="Duration (seconds)",
935
- minimum=1.0, maximum=30.0, value=10.0, step=0.1,
936
- )
937
-
938
- with gr.Row():
939
- seed = gr.Number(label="Seed", value=42, precision=0, minimum=0, maximum=MAX_SEED)
940
- randomize_seed = gr.Checkbox(label="Randomize Seed", value=True)
941
-
942
- with gr.Row():
943
- high_res = gr.Checkbox(label="High Resolution", value=True)
944
- enhance_prompt = gr.Checkbox(label="Enhance Prompt", value=False)
945
-
946
- with gr.Row():
947
- width = gr.Number(label="Width", value=1536, precision=0)
948
- height = gr.Number(label="Height", value=1024, precision=0)
949
-
950
- with gr.Row():
951
- num_inference_steps = gr.Slider(
952
- label="Stage 1 Inference Steps",
953
- minimum=2, maximum=16, value=8, step=1,
954
- info="Higher = more quality but slower (Stage 2 uses fixed 3 steps)"
955
- )
956
-
957
- generate_btn = gr.Button("Generate Video", variant="primary", size="lg")
958
-
959
- with gr.Accordion("Advanced Settings", open=False):
960
- gr.Markdown("### Video Guidance Parameters")
961
-
962
- with gr.Row():
963
- video_cfg_scale = gr.Slider(
964
- label="Video CFG Scale", minimum=1.0, maximum=10.0, value=1.0, step=0.1
965
- )
966
- video_stg_scale = gr.Slider(
967
- label="Video STG Scale", minimum=0.0, maximum=2.0, value=0.0, step=0.1
968
- )
969
-
970
- with gr.Row():
971
- video_rescale_scale = gr.Slider(
972
- label="Video Rescale", minimum=0.0, maximum=2.0, value=0.45, step=0.1
973
- )
974
- video_a2v_scale = gr.Slider(
975
- label="A2V Scale", minimum=0.0, maximum=5.0, value=3.0, step=0.1
976
- )
977
-
978
- gr.Markdown("### Audio Guidance Parameters")
979
-
980
- with gr.Row():
981
- audio_cfg_scale = gr.Slider(
982
- label="Audio CFG Scale", minimum=1.0, maximum=15.0, value=1.0, step=0.1
983
- )
984
- audio_stg_scale = gr.Slider(
985
- label="Audio STG Scale", minimum=0.0, maximum=2.0, value=0.0, step=0.1
986
- )
987
-
988
- with gr.Row():
989
- audio_rescale_scale = gr.Slider(
990
- label="Audio Rescale", minimum=0.0, maximum=2.0, value=1.0, step=0.1
991
- )
992
- audio_v2a_scale = gr.Slider(
993
- label="V2A Scale", minimum=0.0, maximum=5.0, value=3.0, step=0.1
994
- )
995
- with gr.Row():
996
- input_audio = gr.Audio(label="Audio Input (Optional)", type="filepath")
997
-
998
- # RIGHT SIDE: Output and LoRA
999
- with gr.Column():
1000
- output_video = gr.Video(label="Generated Video", autoplay=False)
1001
-
1002
- gpu_duration = gr.Slider(
1003
- label="ZeroGPU duration (seconds)",
1004
- minimum=30.0, maximum=240.0, value=90.0, step=1.0,
1005
- info="Increase for longer videos, higher resolution, or LoRA usage"
1006
- )
1007
-
1008
- gr.Markdown("### LoRA Adapter Strengths")
1009
- gr.Markdown("Set to 0 to disable, then click 'Prepare LoRA Cache'")
1010
-
1011
- with gr.Row():
1012
- pose_strength = gr.Slider(label="Anthro Enhancer", minimum=0.0, maximum=2.0, value=0.0, step=0.01)
1013
- gr.Markdown("") # Spacer for alignment
1014
-
1015
- with gr.Row():
1016
- general_strength = gr.Slider(label="Reasoning Enhancer", minimum=0.0, maximum=2.0, value=0.0, step=0.01)
1017
- motion_strength = gr.Slider(label="Anthro Posing", minimum=0.0, maximum=2.0, value=0.0, step=0.01)
1018
-
1019
- with gr.Row():
1020
- dreamlay_strength = gr.Slider(label="Dreamlay", minimum=0.0, maximum=2.0, value=0.0, step=0.01)
1021
- mself_strength = gr.Slider(label="Mself", minimum=0.0, maximum=2.0, value=0.0, step=0.01)
1022
-
1023
- with gr.Row():
1024
- dramatic_strength = gr.Slider(label="Dramatic", minimum=0.0, maximum=2.0, value=0.0, step=0.01)
1025
- fluid_strength = gr.Slider(label="Fluid Helper", minimum=0.0, maximum=2.0, value=0.0, step=0.01)
1026
-
1027
- with gr.Row():
1028
- liquid_strength = gr.Slider(label="Liquid Helper", minimum=0.0, maximum=2.0, value=0.0, step=0.01)
1029
- demopose_strength = gr.Slider(label="Audio Helper", minimum=0.0, maximum=2.0, value=0.0, step=0.01)
1030
-
1031
- with gr.Row():
1032
- voice_strength = gr.Slider(label="Voice Helper", minimum=0.0, maximum=2.0, value=0.0, step=0.01)
1033
- realism_strength = gr.Slider(label="Anthro Realism", minimum=0.0, maximum=2.0, value=0.0, step=0.01)
1034
-
1035
- with gr.Row():
1036
- transition_strength = gr.Slider(label="POV", minimum=0.0, maximum=2.0, value=0.0, step=0.01)
1037
- physics_strength = gr.Slider(label="Physics", minimum=0.0, maximum=2.0, value=0.0, step=0.01)
1038
- with gr.Row():
1039
- reasoning_strength = gr.Slider(label="Official Reasoning", minimum=0.0, maximum=2.0, value=0.0, step=0.01)
1040
- twostep_strength = gr.Slider(label="Two Step Reasoning", minimum=0.0, maximum=2.0, value=0.0, step=0.01)
1041
-
1042
- prepare_lora_btn = gr.Button("Prepare / Load LoRA Cache", variant="secondary")
1043
- lora_status = gr.Textbox(
1044
- label="LoRA Cache Status",
1045
- value="No LoRA state prepared yet.",
1046
- interactive=False,
1047
- )
1048
-
1049
- # Event handlers
1050
- first_image.change(fn=on_image_upload, inputs=[first_image, last_image, high_res], outputs=[width, height])
1051
- last_image.change(fn=on_image_upload, inputs=[first_image, last_image, high_res], outputs=[width, height])
1052
- high_res.change(fn=on_highres_toggle, inputs=[first_image, last_image, high_res], outputs=[width, height])
1053
-
1054
- prepare_lora_btn.click(
1055
- fn=prepare_lora_cache,
1056
- inputs=[pose_strength, general_strength, motion_strength, dreamlay_strength,
1057
- mself_strength, dramatic_strength, fluid_strength, liquid_strength,
1058
- demopose_strength, voice_strength, realism_strength, transition_strength, physics_strength, reasoning_strength, twostep_strength],
1059
- outputs=[lora_status],
1060
- )
1061
-
1062
- generate_btn.click(
1063
- fn=generate_video,
1064
- inputs=[
1065
- first_image, last_image, input_audio, prompt, negative_prompt, duration, gpu_duration,
1066
- enhance_prompt, seed, randomize_seed, height, width,
1067
- video_cfg_scale, video_stg_scale, video_rescale_scale, video_a2v_scale,
1068
- audio_cfg_scale, audio_stg_scale, audio_rescale_scale, audio_v2a_scale,
1069
- pose_strength, general_strength, motion_strength,
1070
- dreamlay_strength, mself_strength, dramatic_strength, fluid_strength,
1071
- liquid_strength, demopose_strength, voice_strength, realism_strength,
1072
- transition_strength, physics_strength, reasoning_strength, twostep_strength, num_inference_steps,
1073
- ],
1074
- outputs=[output_video, seed],
1075
- )
1076
-
1077
-
1078
- if __name__ == "__main__":
1079
- demo.queue().launch(theme=gr.themes.Citrus(), css=css, mcp_server=False)