45.1 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Video generation","local":"video-generation","sections":[{"title":"Pipeline parameters","local":"pipeline-parameters","sections":[{"title":"num_frames","local":"numframes","sections":[],"depth":3},{"title":"guidance_scale","local":"guidancescale","sections":[],"depth":3},{"title":"negative_prompt","local":"negativeprompt","sections":[],"depth":3}],"depth":2},{"title":"Reduce memory usage","local":"reduce-memory-usage","sections":[],"depth":2},{"title":"Inference speed","local":"inference-speed","sections":[],"depth":2}],"depth":1}">
	<link href="/docs/diffusers/pr_11686/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/entry/start.2b9667fb.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/chunks/scheduler.8c3d61f6.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/chunks/singletons.756349ae.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/chunks/index.0997d446.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/chunks/paths.8d5937da.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/entry/app.a2a6117e.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/chunks/index.da70eac4.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/nodes/0.a31d0923.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/nodes/312.58dedc41.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/chunks/Tip.1d9b8c37.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/chunks/CodeBlock.a9c4becf.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/chunks/getInferenceSnippets.d00e08ac.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/chunks/HfOption.6ab18950.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Video generation","local":"video-generation","sections":[{"title":"Pipeline parameters","local":"pipeline-parameters","sections":[{"title":"num_frames","local":"numframes","sections":[],"depth":3},{"title":"guidance_scale","local":"guidancescale","sections":[],"depth":3},{"title":"negative_prompt","local":"negativeprompt","sections":[],"depth":3}],"depth":2},{"title":"Reduce memory usage","local":"reduce-memory-usage","sections":[],"depth":2},{"title":"Inference speed","local":"inference-speed","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="video-generation" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#video-generation"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Video generation</span></h1> <p data-svelte-h="svelte-1bo3eh2">Video generation models extend image generation (can be considered a 1-frame video) to also process data related to space and time. Making sure all this data - text, space, time - remain consistent and aligned from frame-to-frame is a big challenge in generating long and high-resolution videos.</p> <p data-svelte-h="svelte-i9eaqp">Modern video models tackle this challenge with the diffusion transformer (DiT) architecture. This reduces computational costs and allows more efficient scaling to larger and higher-quality image and video data.</p> <p data-svelte-h="svelte-1aaeavb">Check out what some of these video models are capable of below.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Wan2.1 </div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">HunyuanVideo </div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">LTX-Video </div></div> <div class="language-select"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># pip install ftfy</span>
	<span class="hljs-keyword">import</span> torch
	<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
	<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel, WanPipeline
	<span class="hljs-keyword">from</span> diffusers.hooks.group_offloading <span class="hljs-keyword">import</span> apply_group_offloading
	<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_image
	<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> UMT5EncoderModel

	text_encoder = UMT5EncoderModel.from_pretrained(<span class="hljs-string">"Wan-AI/Wan2.1-T2V-14B-Diffusers"</span>, subfolder=<span class="hljs-string">"text_encoder"</span>, torch_dtype=torch.bfloat16)
	vae = AutoModel.from_pretrained(<span class="hljs-string">"Wan-AI/Wan2.1-T2V-14B-Diffusers"</span>, subfolder=<span class="hljs-string">"vae"</span>, torch_dtype=torch.float32)
	transformer = AutoModel.from_pretrained(<span class="hljs-string">"Wan-AI/Wan2.1-T2V-14B-Diffusers"</span>, subfolder=<span class="hljs-string">"transformer"</span>, torch_dtype=torch.bfloat16)

	<span class="hljs-comment"># group-offloading</span>
	onload_device = torch.device(<span class="hljs-string">"cuda"</span>)
	offload_device = torch.device(<span class="hljs-string">"cpu"</span>)
	apply_group_offloading(text_encoder,
	onload_device=onload_device,
	offload_device=offload_device,
	offload_type=<span class="hljs-string">"block_level"</span>,
	num_blocks_per_group=<span class="hljs-number">4</span>
	)
	transformer.enable_group_offload(
	onload_device=onload_device,
	offload_device=offload_device,
	offload_type=<span class="hljs-string">"leaf_level"</span>,
	use_stream=<span class="hljs-literal">True</span>
	)

	pipeline = WanPipeline.from_pretrained(
	<span class="hljs-string">"Wan-AI/Wan2.1-T2V-14B-Diffusers"</span>,
	vae=vae,
	transformer=transformer,
	text_encoder=text_encoder,
	torch_dtype=torch.bfloat16
	)
	pipeline.to(<span class="hljs-string">"cuda"</span>)

	prompt = <span class="hljs-string">"""
	The camera rushes from far to near in a low-angle shot,
	revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in
	for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground.
	Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts dynamic
	shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
	"""</span>
	negative_prompt = <span class="hljs-string">"""
	Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality,
	low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured,
	misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards
	"""</span>

	output = pipeline(
	prompt=prompt,
	negative_prompt=negative_prompt,
	num_frames=<span class="hljs-number">81</span>,
	guidance_scale=<span class="hljs-number">5.0</span>,
	).frames[<span class="hljs-number">0</span>]
	export_to_video(output, <span class="hljs-string">"output.mp4"</span>, fps=<span class="hljs-number">16</span>)<!-- HTML_TAG_END --></pre></div> </div> <p data-svelte-h="svelte-1q9es90">This guide will cover video generation basics such as which parameters to configure and how to reduce their memory usage.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-qd6kc4">If you’re interested in learning more about how to use a specific model, please refer to their pipeline API model card.</p></div> <h2 class="relative group"><a id="pipeline-parameters" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#pipeline-parameters"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Pipeline parameters</span></h2> <p data-svelte-h="svelte-7qv10n">There are several parameters to configure in the pipeline that’ll affect video generation quality or speed. Experimenting with different parameter values is important for discovering the appropriate quality and speed tradeoff.</p> <h3 class="relative group"><a id="numframes" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#numframes"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>num_frames</span></h3> <p data-svelte-h="svelte-nhumen">A frame is a still image that is played in a sequence of other frames to create motion or a video. Control the number of frames generated per second with <code>num_frames</code>. Increasing <code>num_frames</code> increases perceived motion smoothness and visual coherence, making it especially important for videos with dynamic content. A higher <code>num_frames</code> value also increases video duration.</p> <p data-svelte-h="svelte-iv2t3b">Some video models require more specific <code>num_frames</code> values for inference. For example, <a href="/docs/diffusers/pr_11686/en/api/pipelines/hunyuan_video#diffusers.HunyuanVideoPipeline">HunyuanVideoPipeline</a> recommends calculating the <code>num_frames</code> with <code>(4 * num_frames) +1</code>. Always check a pipelines API model card to see if there is a recommended value.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> torch
	<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> LTXPipeline
	<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video

	pipeline = LTXPipeline.from_pretrained(
	<span class="hljs-string">"Lightricks/LTX-Video"</span>, torch_dtype=torch.bfloat16
	).to(<span class="hljs-string">"cuda"</span>)

	prompt = <span class="hljs-string">"""
	A woman with long brown hair and light skin smiles at another woman with long blonde hair. The woman
	with brown hair wears a black jacket and has a small, barely noticeable mole on her right cheek. The
	camera angle is a close-up, focused on the woman with brown hair's face. The lighting is warm and
	natural, likely from the setting sun, casting a soft glow on the scene. The scene appears to be
	real-life footage
	"""</span>

	video = pipeline(
	prompt=prompt,
	negative_prompt=negative_prompt,
	width=<span class="hljs-number">768</span>,
	height=<span class="hljs-number">512</span>,
	num_frames=<span class="hljs-number">161</span>,
	decode_timestep=<span class="hljs-number">0.03</span>,
	decode_noise_scale=<span class="hljs-number">0.025</span>,
	num_inference_steps=<span class="hljs-number">50</span>,
	).frames[<span class="hljs-number">0</span>]
	export_to_video(video, <span class="hljs-string">"output.mp4"</span>, fps=<span class="hljs-number">24</span>)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="guidancescale" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#guidancescale"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>guidance_scale</span></h3> <p data-svelte-h="svelte-7ligo6">Guidance scale or “cfg” controls how closely the generated frames adhere to the input conditioning (text, image or both). Increasing <code>guidance_scale</code> generates frames that resemble the input conditions more closely and includes finer details, but risk introducing artifacts and reducing output diversity. Lower <code>guidance_scale</code> values encourages looser prompt adherence and increased output variety, but details may not be as great. If it’s too low, it may ignore your prompt entirely and generate random noise.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> torch
	<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> CogVideoXPipeline, CogVideoXTransformer3DModel
	<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video

	pipeline = CogVideoXPipeline.from_pretrained(
	<span class="hljs-string">"THUDM/CogVideoX-2b"</span>,
	torch_dtype=torch.float16
	).to(<span class="hljs-string">"cuda"</span>)

	prompt = <span class="hljs-string">"""
	A detailed wooden toy ship with intricately carved masts and sails is seen gliding smoothly over
	a plush, blue carpet that mimics the waves of the sea. The ship's hull is painted a rich brown,
	with tiny windows. The carpet, soft and textured, provides a perfect backdrop, resembling an
	oceanic expanse. Surrounding the ship are various other toys and children's items, hinting at
	a playful environment. The scene captures the innocence and imagination of childhood,
	with the toy ship's journey symbolizing endless adventures in a whimsical, indoor setting.
	"""</span>

	video = pipeline(
	prompt=prompt,
	guidance_scale=<span class="hljs-number">6</span>,
	num_inference_steps=<span class="hljs-number">50</span>
	).frames[<span class="hljs-number">0</span>]
	export_to_video(video, <span class="hljs-string">"output.mp4"</span>, fps=<span class="hljs-number">8</span>)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="negativeprompt" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#negativeprompt"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>negative_prompt</span></h3> <p data-svelte-h="svelte-11664c2">A negative prompt is useful for excluding things you don’t want to see in the generated video. It is commonly used to refine the quality and alignment of the generated video by pushing the model away from undesirable elements like “blurry, distorted, ugly”. This can create cleaner and more focused videos.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># pip install ftfy</span>
	<span class="hljs-keyword">import</span> torch
	<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> WanPipeline
	<span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler
	<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video

	vae = AutoencoderKLWan.from_pretrained(
	<span class="hljs-string">"Wan-AI/Wan2.1-T2V-14B-Diffusers"</span>, subfolder=<span class="hljs-string">"vae"</span>, torch_dtype=torch.float32
	)
	pipeline = WanPipeline.from_pretrained(
	<span class="hljs-string">"Wan-AI/Wan2.1-T2V-14B-Diffusers"</span>, vae=vae, torch_dtype=torch.bfloat16
	)
	pipeline.scheduler = UniPCMultistepScheduler.from_config(
	pipeline.scheduler.config, flow_shift=<span class="hljs-number">5.0</span>
	)
	pipeline.to(<span class="hljs-string">"cuda"</span>)

	pipeline.load_lora_weights(<span class="hljs-string">"benjamin-paine/steamboat-willie-14b"</span>, adapter_name=<span class="hljs-string">"steamboat-willie"</span>)
	pipeline.set_adapters(<span class="hljs-string">"steamboat-willie"</span>)

	pipeline.enable_model_cpu_offload()

	<span class="hljs-comment"># use "steamboat willie style" to trigger the LoRA</span>
	prompt = <span class="hljs-string">"""
	steamboat willie style, golden era animation, The camera rushes from far to near in a low-angle shot,
	revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in
	for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground.
	Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts
	dynamic shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
	"""</span>

	output = pipeline(
	prompt=prompt,
	num_frames=<span class="hljs-number">81</span>,
	guidance_scale=<span class="hljs-number">5.0</span>,
	).frames[<span class="hljs-number">0</span>]
	export_to_video(output, <span class="hljs-string">"output.mp4"</span>, fps=<span class="hljs-number">16</span>)<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="reduce-memory-usage" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#reduce-memory-usage"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Reduce memory usage</span></h2> <p data-svelte-h="svelte-ha052p">Recent video models like <a href="/docs/diffusers/pr_11686/en/api/pipelines/hunyuan_video#diffusers.HunyuanVideoPipeline">HunyuanVideoPipeline</a> and <a href="/docs/diffusers/pr_11686/en/api/pipelines/wan#diffusers.WanPipeline">WanPipeline</a>, which have 10B+ parameters, require a lot of memory and it often exceeds the memory availabe on consumer hardware. Diffusers offers several techniques for reducing the memory requirements of these large models.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-1ih8c3a">Refer to the <a href="../optimization/memory">Reduce memory usage</a> guide for more details about other memory saving techniques.</p></div> <p data-svelte-h="svelte-1rejsfz">One of these techniques is <a href="../optimization/memory#group-offloading">group-offloading</a>, which offloads groups of internal model layers (such as <code>torch.nn.Sequential</code>) to the CPU when it isn’t being used. These layers are only loaded when they’re needed for computation to avoid storing <strong>all</strong> the model components on the GPU. For a 14B parameter model like <a href="/docs/diffusers/pr_11686/en/api/pipelines/wan#diffusers.WanPipeline">WanPipeline</a>, group-offloading can lower the required memory to ~13GB of VRAM.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># pip install ftfy</span>
	<span class="hljs-keyword">import</span> torch
	<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
	<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel, WanPipeline
	<span class="hljs-keyword">from</span> diffusers.hooks.group_offloading <span class="hljs-keyword">import</span> apply_group_offloading
	<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_image
	<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> UMT5EncoderModel

	text_encoder = UMT5EncoderModel.from_pretrained(<span class="hljs-string">"Wan-AI/Wan2.1-T2V-14B-Diffusers"</span>, subfolder=<span class="hljs-string">"text_encoder"</span>, torch_dtype=torch.bfloat16)
	vae = AutoModel.from_pretrained(<span class="hljs-string">"Wan-AI/Wan2.1-T2V-14B-Diffusers"</span>, subfolder=<span class="hljs-string">"vae"</span>, torch_dtype=torch.float32)
	transformer = AutoModel.from_pretrained(<span class="hljs-string">"Wan-AI/Wan2.1-T2V-14B-Diffusers"</span>, subfolder=<span class="hljs-string">"transformer"</span>, torch_dtype=torch.bfloat16)

	<span class="hljs-comment"># group-offloading</span>
	onload_device = torch.device(<span class="hljs-string">"cuda"</span>)
	offload_device = torch.device(<span class="hljs-string">"cpu"</span>)
	apply_group_offloading(text_encoder,
	onload_device=onload_device,
	offload_device=offload_device,
	offload_type=<span class="hljs-string">"block_level"</span>,
	num_blocks_per_group=<span class="hljs-number">4</span>
	)
	transformer.enable_group_offload(
	onload_device=onload_device,
	offload_device=offload_device,
	offload_type=<span class="hljs-string">"leaf_level"</span>,
	use_stream=<span class="hljs-literal">True</span>
	)

	pipeline = WanPipeline.from_pretrained(
	<span class="hljs-string">"Wan-AI/Wan2.1-T2V-14B-Diffusers"</span>,
	vae=vae,
	transformer=transformer,
	text_encoder=text_encoder,
	torch_dtype=torch.bfloat16
	)
	pipeline.to(<span class="hljs-string">"cuda"</span>)

	prompt = <span class="hljs-string">"""
	The camera rushes from far to near in a low-angle shot,
	revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in
	for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground.
	Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts dynamic
	shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
	"""</span>
	negative_prompt = <span class="hljs-string">"""
	Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality,
	low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured,
	misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards
	"""</span>

	output = pipeline(
	prompt=prompt,
	negative_prompt=negative_prompt,
	num_frames=<span class="hljs-number">81</span>,
	guidance_scale=<span class="hljs-number">5.0</span>,
	).frames[<span class="hljs-number">0</span>]
	export_to_video(output, <span class="hljs-string">"output.mp4"</span>, fps=<span class="hljs-number">16</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-8hypvu">Another option for reducing memory is to consider quantizing a model, which stores the model weights in a lower precision data type. However, quantization may impact video quality depending on the specific video model. Refer to the quantization <a href="../quantization/overview">Overivew</a> to learn more about the different supported quantization backends.</p> <p data-svelte-h="svelte-1b52ou4">The example below uses <a href="../quantization/bitsandbytes">bitsandbytes</a> to quantize a model.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># pip install ftfy</span>

	<span class="hljs-keyword">import</span> torch
	<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> WanPipeline
	<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel, WanPipeline
	<span class="hljs-keyword">from</span> diffusers.quantizers <span class="hljs-keyword">import</span> PipelineQuantizationConfig
	<span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler
	<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> UMT5EncoderModel
	<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video

	<span class="hljs-comment"># quantize transformer and text encoder weights with bitsandbytes</span>
	pipeline_quant_config = PipelineQuantizationConfig(
	quant_backend=<span class="hljs-string">"bitsandbytes_4bit"</span>,
	quant_kwargs={<span class="hljs-string">"load_in_4bit"</span>: <span class="hljs-literal">True</span>},
	components_to_quantize=[<span class="hljs-string">"transformer"</span>, <span class="hljs-string">"text_encoder"</span>]
	)

	vae = AutoModel.from_pretrained(
	<span class="hljs-string">"Wan-AI/Wan2.1-T2V-14B-Diffusers"</span>, subfolder=<span class="hljs-string">"vae"</span>, torch_dtype=torch.float32
	)
	pipeline = WanPipeline.from_pretrained(
	<span class="hljs-string">"Wan-AI/Wan2.1-T2V-14B-Diffusers"</span>, vae=vae, quantization_config=pipeline_quant_config, torch_dtype=torch.bfloat16
	)
	pipeline.scheduler = UniPCMultistepScheduler.from_config(
	pipeline.scheduler.config, flow_shift=<span class="hljs-number">5.0</span>
	)
	pipeline.to(<span class="hljs-string">"cuda"</span>)

	pipeline.load_lora_weights(<span class="hljs-string">"benjamin-paine/steamboat-willie-14b"</span>, adapter_name=<span class="hljs-string">"steamboat-willie"</span>)
	pipeline.set_adapters(<span class="hljs-string">"steamboat-willie"</span>)

	pipeline.enable_model_cpu_offload()

	<span class="hljs-comment"># use "steamboat willie style" to trigger the LoRA</span>
	prompt = <span class="hljs-string">"""
	steamboat willie style, golden era animation, The camera rushes from far to near in a low-angle shot,
	revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in
	for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground.
	Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts
	dynamic shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
	"""</span>

	output = pipeline(
	prompt=prompt,
	num_frames=<span class="hljs-number">81</span>,
	guidance_scale=<span class="hljs-number">5.0</span>,
	).frames[<span class="hljs-number">0</span>]
	export_to_video(output, <span class="hljs-string">"output.mp4"</span>, fps=<span class="hljs-number">16</span>)<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="inference-speed" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#inference-speed"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Inference speed</span></h2> <p data-svelte-h="svelte-1s92ub6"><a href="https://pytorch.org/tutorials/intermediate/torch_compile_tutorial_.html" rel="nofollow">torch.compile</a> can speedup inference by using optimized kernels. Compilation takes longer the first time, but once compiled, it is much faster. It is best to compile the pipeline once, and then use the pipeline multiple times without changing anything. A change, such as in the image size, triggers recompilation.</p> <p data-svelte-h="svelte-113cz6n">The example below compiles the transformer in the pipeline and uses the <code>"max-autotune"</code> mode to maximize performance.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> torch
	<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> CogVideoXPipeline, CogVideoXTransformer3DModel
	<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video

	pipeline = CogVideoXPipeline.from_pretrained(
	<span class="hljs-string">"THUDM/CogVideoX-2b"</span>,
	torch_dtype=torch.float16
	).to(<span class="hljs-string">"cuda"</span>)

	<span class="hljs-comment"># torch.compile</span>
	pipeline.transformer.to(memory_format=torch.channels_last)
	pipeline.transformer = torch.<span class="hljs-built_in">compile</span>(
	pipeline.transformer, mode=<span class="hljs-string">"max-autotune"</span>, fullgraph=<span class="hljs-literal">True</span>
	)

	prompt = <span class="hljs-string">"""
	A detailed wooden toy ship with intricately carved masts and sails is seen gliding smoothly over a plush, blue carpet that mimics the waves of the sea.
	The ship's hull is painted a rich brown, with tiny windows. The carpet, soft and textured, provides a perfect backdrop, resembling an oceanic expanse.
	Surrounding the ship are various other toys and children's items, hinting at a playful environment. The scene captures the innocence and imagination of childhood,
	with the toy ship's journey symbolizing endless adventures in a whimsical, indoor setting.
	"""</span>

	video = pipeline(
	prompt=prompt,
	guidance_scale=<span class="hljs-number">6</span>,
	num_inference_steps=<span class="hljs-number">50</span>
	).frames[<span class="hljs-number">0</span>]
	export_to_video(video, <span class="hljs-string">"output.mp4"</span>, fps=<span class="hljs-number">8</span>)<!-- HTML_TAG_END --></pre></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/using-diffusers/text-img2vid.md" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span data-svelte-h="svelte-x0xyl0">></span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_o82a48 = {
	assets: "/docs/diffusers/pr_11686/en",
	base: "/docs/diffusers/pr_11686/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/diffusers/pr_11686/en/_app/immutable/entry/start.2b9667fb.js"),
	import("/docs/diffusers/pr_11686/en/_app/immutable/entry/app.a2a6117e.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 312],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 45.1 kB
Xet hash:: 8d5efab08060de904e4f12d2572a22cebb1bfd9c832c9591cd4d753f13a2539d

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.