Upload 4 files

README.md

@@ -1,13 +1,89 @@
 ---
-title: ControlFoley
-emoji: 🏆
-colorFrom: blue
-colorTo: indigo
+title: ControlFoley Video to Audio
+emoji: 🎬
+colorFrom: indigo
+colorTo: purple
 sdk: gradio
-sdk_version: 6.25.0
-python_version: '3.12'
+sdk_version: 6.22.0
 app_file: app.py
 pinned: false
+python_version: 3.10.13
+license: apache-2.0
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# ControlFoley Video → Foley Audio — ZeroGPU Space
+
+A Gradio/Hugging Face ZeroGPU wrapper around the official
+[Xiaomi Research ControlFoley](https://github.com/xiaomi-research/controlfoley)
+inference code.
+
+## What this Space does
+
+- Upload a video.
+- Generate synchronized Foley/audio from the video.
+- Optionally guide generation with a text prompt.
+- Preview and download the generated FLAC.
+- Preview and download the video with the generated soundtrack.
+- Includes the official `assets/001.mp4` skateboard example.
+
+## Hardware
+
+After creating the Space, open **Settings → Hardware** and select **ZeroGPU**.
+
+The app uses:
+
+```python
+@spaces.GPU(size="xlarge", duration=...)
+```
+
+`xlarge` is intentionally selected because ControlFoley loads a large main
+checkpoint plus multiple visual/audio feature models. If your testing confirms
+that the complete model stack fits comfortably in 48 GB VRAM, change
+`size="xlarge"` to `size="large"` to reduce ZeroGPU quota consumption.
+
+## Model downloads
+
+On first startup the app downloads:
+
+- Official ControlFoley source from GitHub.
+- ControlFoley `weights/*`.
+- ControlFoley `ext_weights/*`.
+- Additional upstream pretrained components requested by ControlFoley
+  (for example CLIP/MusicGen) through their normal loaders.
+
+The official ControlFoley model repository is large, so the first cold start
+will take significantly longer than later cached starts.
+
+## Test
+
+Use the bundled sample and this prompt:
+
+```text
+the skateboard wheels scraping and grinding on the ground.
+```
+
+Recommended first test:
+
+- Duration: `8 s`
+- CFG: `4.5`
+- Steps: `25`
+- Seed: `42`
+
+For a quicker smoke test, use `10–15` steps.
+
+## Licensing
+
+The upstream **source code** repository is Apache-2.0.
+
+The **ControlFoley model weights** on Hugging Face are published under
+**CC BY-NC 4.0**. That means the model weights are intended for non-commercial
+use unless you obtain different permission from the model owner.
+
+Review the upstream licenses before publishing or using this Space in a
+commercial context.
+
+## Upstream
+
+- GitHub: `xiaomi-research/controlfoley`
+- Model: `YJX-Xiaomi/ControlFoley`
+- Paper: *ControlFoley: Unified and Controllable Video-to-Audio Generation with Cross-Modal Conflict Handling*

app.py

@@ -0,0 +1,369 @@
+# ControlFoley Gradio app for Hugging Face ZeroGPU
+# Based on: https://github.com/xiaomi-research/controlfoley
+#
+# IMPORTANT: `spaces` must be imported before torch on ZeroGPU.
+import spaces
+
+import os
+import sys
+import time
+import shutil
+import logging
+import subprocess
+import tempfile
+from pathlib import Path
+
+import gradio as gr
+from huggingface_hub import snapshot_download
+
+APP_DIR = Path(__file__).resolve().parent
+SOURCE_DIR = APP_DIR / "upstream_controlfoley"
+MODEL_DIR = APP_DIR / "model_weights"
+OUTPUT_DIR = APP_DIR / "outputs"
+ASSET_DIR = APP_DIR / "sample_assets"
+
+UPSTREAM_REPO = "https://github.com/xiaomi-research/controlfoley.git"
+MODEL_REPO = "YJX-Xiaomi/ControlFoley"
+
+OUTPUT_DIR.mkdir(exist_ok=True)
+ASSET_DIR.mkdir(exist_ok=True)
+
+
+def clone_upstream():
+    """Fetch the official ControlFoley inference source on first startup."""
+    if (SOURCE_DIR / "controlfoley").exists():
+        return
+
+    print("Downloading official ControlFoley source...")
+    try:
+        subprocess.run(
+            ["git", "clone", "--depth", "1", UPSTREAM_REPO, str(SOURCE_DIR)],
+            check=True,
+        )
+    except Exception as exc:
+        raise RuntimeError(
+            "Could not clone the official ControlFoley repository. "
+            "Check that git/network access is available in the Space."
+        ) from exc
+
+
+def download_model_files():
+    """Download only the checkpoint folders needed by official inference."""
+    print("Downloading ControlFoley model files...")
+    snapshot_download(
+        repo_id=MODEL_REPO,
+        local_dir=str(MODEL_DIR),
+        allow_patterns=["weights/*", "ext_weights/*"],
+    )
+
+
+def download_sample_video():
+    """Use the official skateboard example from the ControlFoley repo."""
+    sample_path = ASSET_DIR / "001.mp4"
+    if sample_path.exists():
+        return sample_path
+
+    src = SOURCE_DIR / "assets" / "001.mp4"
+    if not src.exists():
+        raise RuntimeError("Official sample video assets/001.mp4 was not found.")
+    shutil.copy2(src, sample_path)
+    return sample_path
+
+
+# Bootstrap source and model files before importing ControlFoley.
+clone_upstream()
+download_model_files()
+
+# The upstream code imports `lib` as a top-level module.
+sys.path.insert(0, str(SOURCE_DIR))
+sys.path.insert(0, str(SOURCE_DIR / "lib"))
+
+import torch
+import torchaudio
+
+from controlfoley.inference_utils import (
+    all_model_cfg,
+    generate,
+    load_video,
+    make_video,
+    setup_eval_logging,
+)
+from controlfoley.audio_model import create_audio_generation_model
+from controlfoley.feature_extractor import FeaturesUtils
+from lib.flow_matching import FlowMatching
+
+setup_eval_logging()
+log = logging.getLogger("controlfoley-space")
+
+torch.backends.cuda.matmul.allow_tf32 = True
+torch.backends.cudnn.allow_tf32 = True
+
+MODEL_CFG = all_model_cfg["large_44k"]
+SEQ_CFG = MODEL_CFG.seq_cfg
+
+# ZeroGPU recommends placing models on CUDA at module scope.
+# Outside @spaces.GPU this uses Hugging Face CUDA emulation.
+print("Loading ControlFoley main network...")
+NET = create_audio_generation_model(MODEL_CFG.model_name).to(
+    "cuda", torch.float32
+).eval()
+NET.load_weights(
+    torch.load(
+        MODEL_DIR / "weights" / "controlfoley.pth",
+        map_location="cuda",
+        weights_only=True,
+    )
+)
+
+print("Loading ControlFoley feature extractors...")
+FEATURES = FeaturesUtils(
+    tod_vae_ckpt=str(MODEL_DIR / "ext_weights" / "v1-44.pth"),
+    synchformer_ckpt=str(MODEL_DIR / "ext_weights" / "synchformer_state_dict.pth"),
+    cav_mae_ckpt=str(MODEL_DIR / "ext_weights" / "cav_mae_st.pth"),
+    clap_ckpt=str(
+        MODEL_DIR / "ext_weights" / "music_speech_audioset_epoch_15_esc_89.98.pt"
+    ),
+    mode=MODEL_CFG.mode,
+    enable_conditions=True,
+    need_vae_encoder=False,
+).to("cuda", torch.float32).eval()
+
+SAMPLE_VIDEO = download_sample_video()
+
+
+def gpu_budget(video_path, prompt, negative_prompt, duration, cfg_strength, steps, seed):
+    # A conservative ZeroGPU reservation. Shorter requests get a smaller reservation.
+    return min(240, max(90, int(45 + float(duration) * 10 + int(steps) * 2)))
+
+
+@spaces.GPU(size="xlarge", duration=gpu_budget)
+@torch.inference_mode()
+def generate_foley(
+    video_path,
+    prompt,
+    negative_prompt,
+    duration,
+    cfg_strength,
+    steps,
+    seed,
+):
+    """Generate synchronized Foley audio for a video with ControlFoley."""
+    if not video_path:
+        raise gr.Error("Please upload a video or select the sample.")
+
+    video_path = Path(video_path)
+    duration = float(duration)
+    cfg_strength = float(cfg_strength)
+    steps = int(steps)
+    seed = int(seed)
+
+    if not 1.0 <= duration <= 8.0:
+        raise gr.Error("Duration must be between 1 and 8 seconds.")
+    if not 5 <= steps <= 30:
+        raise gr.Error("Inference steps must be between 5 and 30.")
+
+    job_dir = Path(tempfile.mkdtemp(prefix="controlfoley_", dir=OUTPUT_DIR))
+    audio_path = job_dir / "generated_foley.flac"
+    video_out_path = job_dir / "video_with_generated_audio.mp4"
+
+    # Load video according to the official preprocessing pipeline.
+    video_info = load_video(video_path, duration)
+    actual_duration = min(duration, float(video_info.total_duration))
+
+    clip_frames = video_info.clip_embeddings.unsqueeze(0)
+    visual_frames = video_info.visual_features.unsqueeze(0)
+    sync_frames = video_info.sync_embeddings.unsqueeze(0)
+
+    # Update temporal sequence lengths for the selected generation duration.
+    SEQ_CFG.total_time_seconds = actual_duration
+    NET.update_seq_lengths(
+        SEQ_CFG.latent_sequence_length,
+        SEQ_CFG.clip_sequence_length,
+        SEQ_CFG.visual_sequence_length,
+        SEQ_CFG.sync_sequence_length,
+    )
+
+    rng = torch.Generator(device="cuda")
+    rng.manual_seed(seed)
+    fm = FlowMatching(
+        min_sigma=0,
+        inference_mode="euler",
+        num_steps=steps,
+    )
+
+    start = time.time()
+
+    audios = generate(
+        clip_frames,
+        visual_frames,
+        sync_frames,
+        None,               # no reference audio
+        None,               # no timbre reference
+        0.0,
+        [prompt or ""],
+        negative_text=[negative_prompt or ""],
+        feature_utils=FEATURES,
+        net=NET,
+        fm=fm,
+        rng=rng,
+        cfg_strength=cfg_strength,
+    )
+
+    audio = audios.float().cpu()[0]
+    torchaudio.save(
+        str(audio_path),
+        audio,
+        SEQ_CFG.audio_sample_rate,
+    )
+
+    # Mux generated audio back onto the input video using the upstream helper.
+    make_video(
+        video_info,
+        video_out_path,
+        audio,
+        sampling_rate=SEQ_CFG.audio_sample_rate,
+    )
+
+    elapsed = time.time() - start
+    status = (
+        f"Generated {actual_duration:.2f}s of synchronized audio in "
+        f"{elapsed:.1f}s. Seed: {seed}. Steps: {steps}."
+    )
+
+    # Help release transient allocations before ZeroGPU returns the GPU.
+    del audios, audio
+    torch.cuda.empty_cache()
+
+    return str(audio_path), str(video_out_path), status
+
+
+TITLE = """
+# 🎬 ControlFoley — Video → Foley Audio
+Generate synchronized sound effects from a silent video using Xiaomi Research's
+**ControlFoley**.
+
+Upload a video, optionally guide the sound with text, and download both the
+generated FLAC and a video with the generated soundtrack.
+"""
+
+INFO = """
+**Modes**
+- Leave the prompt empty for pure **Video-to-Audio (V2A)**.
+- Add a prompt for **Text + Video-to-Audio (TV2A)**.
+- Maximum demo duration is capped at **8 seconds** to keep ZeroGPU usage practical.
+
+**Sample**
+The bundled example is the official ControlFoley skateboard sample. Try:
+`the skateboard wheels scraping and grinding on the ground.`
+"""
+
+with gr.Blocks(title="ControlFoley Video to Audio") as demo:
+    gr.Markdown(TITLE)
+
+    with gr.Row():
+        with gr.Column():
+            video = gr.Video(
+                label="Input video",
+                sources=["upload"],
+                format="mp4",
+            )
+            prompt = gr.Textbox(
+                label="Sound prompt (optional)",
+                placeholder="e.g. skateboard wheels scraping on concrete",
+                value="the skateboard wheels scraping and grinding on the ground.",
+            )
+            negative_prompt = gr.Textbox(
+                label="Negative prompt (optional)",
+                placeholder="e.g. music, speech, crowd noise",
+                value="",
+            )
+
+            with gr.Accordion("Generation settings", open=False):
+                duration = gr.Slider(
+                    minimum=1,
+                    maximum=8,
+                    value=8,
+                    step=0.5,
+                    label="Duration (seconds)",
+                )
+                cfg_strength = gr.Slider(
+                    minimum=1.0,
+                    maximum=8.0,
+                    value=4.5,
+                    step=0.5,
+                    label="CFG strength",
+                )
+                steps = gr.Slider(
+                    minimum=5,
+                    maximum=30,
+                    value=25,
+                    step=1,
+                    label="Inference steps",
+                )
+                seed = gr.Number(
+                    value=42,
+                    precision=0,
+                    label="Seed",
+                )
+
+            generate_btn = gr.Button(
+                "Generate Foley Sound",
+                variant="primary",
+            )
+
+        with gr.Column():
+            audio_out = gr.Audio(
+                label="Generated Foley audio",
+                type="filepath",
+            )
+            video_out = gr.Video(
+                label="Video with generated audio",
+            )
+            status = gr.Markdown()
+
+    gr.Markdown(INFO)
+
+    gr.Examples(
+        examples=[
+            [
+                str(SAMPLE_VIDEO),
+                "the skateboard wheels scraping and grinding on the ground.",
+                "",
+                8,
+                4.5,
+                25,
+                42,
+            ],
+        ],
+        inputs=[
+            video,
+            prompt,
+            negative_prompt,
+            duration,
+            cfg_strength,
+            steps,
+            seed,
+        ],
+        label="Official ControlFoley sample",
+    )
+
+    generate_btn.click(
+        fn=generate_foley,
+        inputs=[
+            video,
+            prompt,
+            negative_prompt,
+            duration,
+            cfg_strength,
+            steps,
+            seed,
+        ],
+        outputs=[
+            audio_out,
+            video_out,
+            status,
+        ],
+        api_name="generate_foley",
+    )
+
+demo.queue(max_size=20).launch()

packages.txt

@@ -0,0 +1,3 @@
+ffmpeg
+git
+libsndfile1

requirements.txt

@@ -0,0 +1,42 @@
+# Hugging Face ZeroGPU currently supports PyTorch 2.8+
+torch==2.8.0
+torchvision==0.23.0
+torchaudio==2.8.0
+
+gradio>=5.50,<7
+spaces
+huggingface_hub>=0.36.0
+
+# ControlFoley runtime dependencies
+numpy==1.26.4
+av==14.0.1
+colorlog==6.9.0
+einops==0.8.0
+omegaconf==2.3.0
+open_clip_torch==2.29.0
+laion_clap==1.1.6
+julius==0.2.7
+librosa==0.11.0
+soundfile==0.13.1
+scipy>=1.13,<1.16
+scikit-learn>=1.5,<1.8
+pandas>=2.2,<2.4
+h5py>=3.11
+timm>=1.0
+transformers==4.38.2
+tokenizers==0.15.2
+sentencepiece>=0.2
+safetensors>=0.4
+torchdiffeq==0.2.5
+torchlibrosa==0.1.0
+nnAudio==0.3.3
+webdataset==1.0.2
+flashy==0.0.2
+dora_search==0.1.12
+submitit==1.5.4
+treetable==0.2.6
+wget==3.2
+ftfy>=6.2
+regex
+requests
+PyYAML