Upload folder using huggingface_hub

README.md

@@ -1,13 +1,18 @@
 ---
-title: Plaguekind Minimax H3
-emoji: 🐠
+title: MiniMax-H3
+emoji: 🎬
 colorFrom: gray
-colorTo: yellow
+colorTo: indigo
 sdk: gradio
 sdk_version: 6.22.0
-python_version: '3.12'
 app_file: app.py
-pinned: false
+short_description: MiniMax-H3 video generation with synchronized audio
+python_version: "3.12"
+startup_duration_timeout: 1h
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# MiniMax-H3
+
+MiniMax-H3 is a 33B parameter state-of-the-art video generation model that produces video and a fully synchronized soundtrack (ambience, foley, speech). This Space runs the FL2VA variant with Int8 weight-only quantization on ZeroGPU.
+
+Based on [Plaguekind/Minimax-H3](https://huggingface.co/Plaguekind/Minimax-H3) (ComfyUI workflow wrapper) and [MiniMaxAI/MiniMax-H3](https://huggingface.co/MiniMaxAI/MiniMax-H3) (original model).

app.py

@@ -0,0 +1,359 @@
+"""MiniMax-H3 video generation with synchronized audio — FL2VA (text / first-last-frame to video+audio).
+
+Based on the diffusers MiniMax-H3 pipeline. Uses Int8 weight-only quantization on both the
+33B transformer and the 32B Qwen3-VL text encoder to fit within ZeroGPU xlarge (96 GB VRAM).
+"""
+
+from __future__ import annotations
+
+import os
+
+# Allocator config for memory pressure (video DiTs have large transient allocations)
+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+
+import spaces  # MUST come before torch / any CUDA-touching import
+import gradio as gr
+
+import tempfile
+import time
+import traceback
+
+MODEL_REPO = os.environ.get("H3_MODEL_REPO", "MiniMaxAI/MiniMax-H3")
+GPU_SIZE = "xlarge"
+
+# Canvas presets matching the original model's supported resolutions
+CANVASES = {
+    # 16:9
+    "960x544 · 16:9 fast": (544, 960),
+    "1024x576 · 16:9 fast": (576, 1024),
+    "1152x640 · 16:9": (640, 1152),
+    "1280x704 · 16:9": (704, 1280),
+    "1344x768 · 16:9 full": (768, 1344),
+    # 9:16
+    "544x960 · 9:16 fast": (960, 544),
+    "640x1152 · 9:16": (1152, 640),
+    "768x1344 · 9:16 full": (1344, 768),
+    # 1:1
+    "544x544 · 1:1 fast": (544, 544),
+    "768x768 · 1:1 full": (768, 768),
+    # 4:3 / 3:4
+    "768x576 · 4:3 fast": (576, 768),
+    "1024x768 · 4:3 full": (768, 1024),
+    "576x768 · 3:4 fast": (768, 576),
+    "768x1024 · 3:4 full": (1024, 768),
+    # 21:9
+    "1152x512 · 21:9 fast": (512, 1152),
+    "1536x672 · 21:9 full": (672, 1536),
+}
+DEFAULT_CANVAS = "960x544 · 16:9 fast"
+FPS, FRAMES_PER_CHUNK, LATENTS_PER_CHUNK = 24, 17, 5
+MIN_UI_DURATION, MAX_UI_DURATION = 2, 10
+
+
+def snap_frames(seconds: float) -> int:
+    """The frame count MiniMax-H3's video VAE can decode: the next 17*n+5 at 24 fps."""
+    frames = max(1, round(float(seconds) * FPS))
+    while frames % FRAMES_PER_CHUNK != LATENTS_PER_CHUNK:
+        frames += 1
+    return frames
+
+
+def lower_duration_floor(seconds: float = MIN_UI_DURATION) -> None:
+    """Let the pipeline generate below its 5 s floor."""
+    from diffusers.modular_pipelines.minimax_h3.modular_pipeline import MiniMaxH3ModularPipeline
+
+    MiniMaxH3ModularPipeline.min_duration = property(lambda self: float(seconds))
+
+
+PIPE = None
+LOAD_ERROR: str | None = None
+LOADED_IN: float | None = None
+
+
+def load_models() -> str | None:
+    """Load the full MiniMax-H3 FL2VA pipeline with Int8 quantization at startup.
+
+    Int8 weight-only quantization roughly halves VRAM for both the 33B transformer and
+    the 32B Qwen3-VL text encoder, bringing the total to ~63 GB — fits in xlarge (96 GB).
+    """
+    global PIPE, LOAD_ERROR, LOADED_IN
+
+    if PIPE is not None or LOAD_ERROR is not None:
+        return LOAD_ERROR
+
+    started = time.time()
+    try:
+        import torch
+        from diffusers import ModularPipeline, MiniMaxH3Transformer3DModel, TorchAoConfig
+        from transformers import Qwen3VLForConditionalGeneration
+        from transformers import TorchAoConfig as TransformersTorchAoConfig
+        from torchao.quantization import Int8WeightOnlyConfig
+
+        lower_duration_floor()
+
+        print(f"[load] loading MiniMax-H3 from {MODEL_REPO} with Int8 quantization ...", flush=True)
+
+        pipe = ModularPipeline.from_pretrained(MODEL_REPO)
+
+        # Quantize the 33B transformer with Int8 weight-only quantization
+        pipe.update_components(
+            transformer=MiniMaxH3Transformer3DModel.from_pretrained(
+                MODEL_REPO,
+                subfolder="transformer",
+                dtype=torch.bfloat16,
+                quantization_config=TorchAoConfig(
+                    Int8WeightOnlyConfig(version=2),
+                    modules_to_not_convert=[
+                        "proj_in", "audio_proj_in", "context_embedder", "time_embedder", "time_proj",
+                        "token_refiner", "norm_out", "proj_out", "audio_proj_out",
+                    ],
+                ),
+                low_cpu_mem_usage=False,
+            ),
+            # Quantize the 32B Qwen3-VL text encoder with Int8 weight-only quantization
+            text_encoder=Qwen3VLForConditionalGeneration.from_pretrained(
+                MODEL_REPO,
+                subfolder="text_encoder",
+                dtype=torch.bfloat16,
+                quantization_config=TransformersTorchAoConfig(
+                    Int8WeightOnlyConfig(version=2),
+                    modules_to_not_convert=[
+                        "model.visual",
+                        "model.language_model.embed_tokens",
+                        "model.language_model.norm",
+                        "lm_head",
+                    ],
+                ),
+            ),
+        )
+        pipe.load_components(dtype=torch.bfloat16)
+
+        # VAEs stay full precision — a bfloat16 audio VAE decodes the soundtrack ~20 dB too quiet.
+        # VAEs are small (~6 GB) so we can pack them at startup; the quantized transformer (~31 GB) and
+        # text encoder (~26 GB) are too large to pack alongside the BF16 download on disk, so they stay
+        # as CPU tensors and move to CUDA on the first GPU call.
+        pipe.vae.to("cuda")
+        pipe.audio_vae.to("cuda")
+
+        # Use cuDNN fused attention (10-20% faster than SDPA, no extra deps)
+        try:
+            pipe.transformer.set_attention_backend("_native_cudnn")
+        except Exception:
+            pipe.transformer.set_attention_backend("sdpa")
+
+        PIPE = pipe
+        LOADED_IN = time.time() - started
+        print(f"[load] ready in {LOADED_IN:.0f}s", flush=True)
+    except Exception as error:
+        traceback.print_exc()
+        LOAD_ERROR = f"**Loading `{MODEL_REPO}` failed** after {time.time() - started:.0f}s: `{type(error).__name__}: {error}`"
+    return LOAD_ERROR
+
+
+def status() -> str:
+    if LOAD_ERROR:
+        return LOAD_ERROR
+    if PIPE is None:
+        return f"Loading `{MODEL_REPO}` (~119 GB BF16, Int8-quantized at load). Watch the Space logs."
+    return f"Ready · Int8 quantized · loaded in {LOADED_IN:.0f}s"
+
+
+# Duration estimation: linear in rows (matmuls) + quadratic (attention) + decode cost
+_DUR_B, _DUR_C = 1.5e-4, 5.0e-9
+_DECODE_BASE, _DECODE_PER_DEFAULT_CANVAS, _DEFAULT_CANVAS_PIXELS = 15, 15, 960 * 544 * 124
+_PAD = 15
+
+
+def get_duration(prompt, image, last_image, height, width, num_frames, steps, seed, *a, **k):
+    """Estimate GPU seconds needed for this request."""
+    height, width, num_frames, steps = int(height), int(width), int(num_frames), int(steps)
+    latent_frames = (num_frames - LATENTS_PER_CHUNK) // FRAMES_PER_CHUNK * LATENTS_PER_CHUNK + 2
+    patches = (height // 32) * (width // 32)
+    rows = latent_frames * patches + (int(image is not None) + int(last_image is not None)) * patches
+    denoise = steps * (_DUR_B * rows + _DUR_C * rows**2)
+    decode = _DECODE_BASE + _DECODE_PER_DEFAULT_CANVAS * (height * width * num_frames) / _DEFAULT_CANVAS_PIXELS
+    # Extra allowance for quantized model being somewhat slower per step + cold-start weight transfer
+    quant_allowance = 30
+    return max(60, int(denoise + decode) + quant_allowance + _PAD)
+
+
+@spaces.GPU(duration=get_duration, size=GPU_SIZE)
+def generate(prompt, image, last_image, height, width, num_frames, steps, seed, progress=gr.Progress(track_tqdm=True)):
+    """Generate video with synchronized audio from text and optional keyframes.
+
+    Args:
+        prompt: Text description of the video to generate.
+        image: Optional first frame image (PIL Image).
+        last_image: Optional last frame image (PIL Image).
+        height: Output video height in pixels.
+        width: Output video width in pixels.
+        num_frames: Number of frames to generate (must be 17*n+5).
+        steps: Number of denoising steps.
+        seed: Random seed for reproducibility.
+    """
+    import torch
+    from diffusers.utils import encode_video
+
+    # Move the quantized transformer and text encoder to CUDA on each cold worker.
+    # They were kept as CPU tensors at startup to avoid exceeding the 150 GB disk
+    # quota (BF16 download ~119 GB + packed copy would be too large).
+    PIPE.transformer.to("cuda")
+    PIPE.text_encoder.to("cuda")
+
+    state = PIPE(
+        prompt=prompt,
+        image=image,
+        last_image=last_image,
+        height=int(height),
+        width=int(width),
+        num_frames=int(num_frames),
+        num_inference_steps=int(steps),
+        generator=torch.Generator("cpu").manual_seed(int(seed)),
+    )
+
+    videos = state.get("videos")
+    audio = state.get("audio")
+    sampling_rate = state.get("sampling_rate")
+
+    directory = os.path.join(tempfile.gettempdir(), "h3-outputs")
+    os.makedirs(directory, exist_ok=True)
+    path = os.path.join(directory, f"h3-{int(time.time() * 1000)}.mp4")
+    encode_video(
+        videos[0],
+        fps=FPS,
+        output_path=path,
+        audio=audio[0].cpu(),
+        audio_sample_rate=sampling_rate,
+    )
+    return path
+
+
+def run_generate(prompt, image_path=None, last_image_path=None, canvas=DEFAULT_CANVAS,
+                 duration=5, steps=28, seed=42, progress=gr.Progress(track_tqdm=True)):
+    """Handle a generation request from the Gradio UI."""
+    if LOAD_ERROR:
+        raise gr.Error(LOAD_ERROR)
+    if PIPE is None:
+        raise gr.Error("The model is still loading. Please wait a moment and try again.")
+    if not prompt or not prompt.strip():
+        raise gr.Error("MiniMax-H3 always takes a prompt, keyframes or not.")
+
+    from PIL import Image, ImageOps
+
+    num_frames = snap_frames(duration)
+    height, width = CANVASES[canvas]
+
+    def keyframe(path):
+        return ImageOps.exif_transpose(Image.open(path)).convert("RGB") if path else None
+
+    first_frame = keyframe(image_path) if image_path else None
+    final_frame = keyframe(last_image_path) if last_image_path else None
+
+    progress(0.05, desc=f"Denoising {steps} steps at {width}x{height}, {num_frames} frames ...")
+    path = generate(prompt, first_frame, final_frame, height, width, num_frames, steps, seed, progress)
+    return path
+
+
+def _fit_keyframe(image_path, current_canvas):
+    """Cover-crop an uploaded keyframe to the closest supported aspect ratio."""
+    if not image_path:
+        return gr.update(), gr.update()
+    from PIL import Image as _Image
+
+    img = _Image.open(image_path)
+    aspect = img.width / img.height
+    fastest = {}
+    for label, (h, w) in CANVASES.items():
+        r = w / h
+        if r not in fastest or w * h < fastest[r][1][0] * fastest[r][1][1]:
+            fastest[r] = (label, (h, w))
+    ratio = min(fastest, key=lambda r: abs(r - aspect))
+    label, (h, w) = fastest[ratio]
+
+    cur_h, cur_w = CANVASES[current_canvas]
+    if abs(cur_w / cur_h - aspect) <= abs(ratio - aspect):
+        label = current_canvas
+        h, w = cur_h, cur_w
+
+    target = w / h
+    if abs(img.width / img.height - target) <= 1e-3:
+        return gr.update(), gr.update(value=label)
+    if img.width / img.height > target:
+        new_w = int(img.height * target)
+        left = (img.width - new_w) // 2
+        img = img.crop((left, 0, left + new_w, img.height))
+    else:
+        new_h = int(img.width / target)
+        top = (img.height - new_h) // 2
+        img = img.crop((0, top, img.width, top + new_h))
+    img.save(image_path)
+    return gr.update(value=image_path), gr.update(value=label)
+
+
+load_models()
+
+INTRO = """# MiniMax-H3
+
+<div align="center">
+  <a href="https://huggingface.co/MiniMaxAI/MiniMax-H3" target="_blank" rel="noopener"><strong>[ model ]</strong></a> &nbsp;
+  <a href="https://www.minimax.io/blog/minimax-h3" target="_blank" rel="noopener"><strong>[ blog ]</strong></a> &nbsp;
+  <a href="https://huggingface.co/Plaguekind/Minimax-H3" target="_blank" rel="noopener"><strong>[ ComfyUI weights ]</strong></a>
+</div>
+
+**MiniMax-H3** is a 33B parameter state-of-the-art video generation model that produces video and a
+fully synchronized soundtrack (ambience, foley, speech). Supports text-to-video and first/last-frame-to-video.
+"""
+
+CSS = """
+.main.fillable {max-width: 1250px !important}
+.dark .gradio-container { color: var(--body-text-color); }
+"""
+
+with gr.Blocks(title="MiniMax-H3") as demo:
+    gr.Markdown(INTRO)
+
+    with gr.Row():
+        with gr.Column():
+            prompt = gr.Textbox(
+                label="Prompt",
+                lines=3,
+                value="A red fox trotting through a snowy pine forest at dawn, snow crunching underfoot",
+            )
+            with gr.Row():
+                image = gr.Image(label="First frame (optional)", type="filepath")
+                last_image = gr.Image(label="Last frame (optional)", type="filepath")
+            run = gr.Button("Generate", variant="primary")
+            with gr.Accordion("Advanced options", open=False):
+                canvas = gr.Dropdown(label="Canvas", choices=list(CANVASES), value=DEFAULT_CANVAS)
+                duration = gr.Slider(label="Duration (s)", minimum=MIN_UI_DURATION, maximum=MAX_UI_DURATION, step=1, value=5)
+                steps = gr.Slider(label="Steps", minimum=10, maximum=40, step=1, value=28)
+                seed = gr.Number(label="Seed", value=42, precision=0)
+
+        with gr.Column():
+            video = gr.Video(label="Video + soundtrack")
+
+    image.upload(_fit_keyframe, [image, canvas], [image, canvas])
+
+    gr.Examples(
+        examples=[
+            ["A red fox trotting through a snowy pine forest at dawn, snow crunching underfoot", None, None, "960x544 · 16:9 fast"],
+            ["A busy night market, neon signs reflecting in puddles, sizzling street food", None, None, "544x960 · 9:16 fast"],
+            ["A cellist playing a slow melody in an empty concert hall", None, None, "544x544 · 1:1 fast"],
+        ],
+        inputs=[prompt, image, last_image, canvas],
+        outputs=[video],
+        fn=run_generate,
+        cache_examples=True,
+        cache_mode="lazy",
+    )
+
+    run.click(
+        run_generate,
+        [prompt, image, last_image, canvas, duration, steps, seed],
+        [video],
+        api_name="generate",
+    )
+
+
+if __name__ == "__main__":
+    demo.queue().launch(show_error=True, theme=gr.themes.Citrus(), css=CSS, mcp_server=True)

requirements.txt

@@ -0,0 +1,16 @@
+# diffusers is installed from the canonical MiniMax-H3 PR
+# https://github.com/huggingface/diffusers/pull/14371 ("Minimax h3 follow up (review & refactor)")
+--extra-index-url https://download.pytorch.org/whl/cu130
+diffusers @ git+https://github.com/huggingface/diffusers.git@665f578278365ea4a3318cb8c9b66ce6c01204b9
+torch==2.11.0
+torchvision==0.26.0
+# The Qwen3-VL processor decides the vision patch count, so a different minor changes the conditioning.
+transformers==5.8.0
+accelerate==1.14.0
+# Int8 weight-only quantization for the 33B transformer and 32B text encoder
+torchao==0.18.0
+# PyAV muxes the generated soundtrack onto the frames (encode_video)
+av
+pillow
+numpy
+safetensors>=0.8.0