17.4 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"CogVideoX","local":"cogvideox","sections":[{"title":"Load model checkpoints","local":"load-model-checkpoints","sections":[],"depth":2},{"title":"Text-to-Video","local":"text-to-video","sections":[],"depth":2},{"title":"Image-to-Video","local":"image-to-video","sections":[],"depth":2}],"depth":1}">
	<link href="/docs/diffusers/pr_10567/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/diffusers/pr_10567/en/_app/immutable/entry/start.5ab964f0.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_10567/en/_app/immutable/chunks/scheduler.8c3d61f6.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_10567/en/_app/immutable/chunks/singletons.1271a703.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_10567/en/_app/immutable/chunks/index.0997d446.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_10567/en/_app/immutable/chunks/paths.af967ee5.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_10567/en/_app/immutable/entry/app.d83dbfce.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_10567/en/_app/immutable/chunks/index.da70eac4.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_10567/en/_app/immutable/nodes/0.bb4a0671.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_10567/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_10567/en/_app/immutable/nodes/234.3b01dcf8.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_10567/en/_app/immutable/chunks/CodeBlock.00a903b3.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_10567/en/_app/immutable/chunks/EditOnGithub.1e64e623.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"CogVideoX","local":"cogvideox","sections":[{"title":"Load model checkpoints","local":"load-model-checkpoints","sections":[],"depth":2},{"title":"Text-to-Video","local":"text-to-video","sections":[],"depth":2},{"title":"Image-to-Video","local":"image-to-video","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="cogvideox" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#cogvideox"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>CogVideoX</span></h1> <p data-svelte-h="svelte-18y6jrj">CogVideoX is a text-to-video generation model focused on creating more coherent videos aligned with a prompt. It achieves this using several methods.</p> <ul data-svelte-h="svelte-pybda1"><li><p>a 3D variational autoencoder that compresses videos spatially and temporally, improving compression rate and video accuracy.</p></li> <li><p>an expert transformer block to help align text and video, and a 3D full attention module for capturing and creating spatially and temporally accurate videos.</p></li></ul> <h2 class="relative group"><a id="load-model-checkpoints" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#load-model-checkpoints"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Load model checkpoints</span></h2> <p data-svelte-h="svelte-1ou5cd0">Model weights may be stored in separate subfolders on the Hub or locally, in which case, you should use the <a href="/docs/diffusers/pr_10567/en/api/pipelines/overview#diffusers.DiffusionPipeline.from_pretrained">from_pretrained()</a> method.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> CogVideoXPipeline, CogVideoXImageToVideoPipeline
	pipe = CogVideoXPipeline.from_pretrained(
	<span class="hljs-string">"THUDM/CogVideoX-2b"</span>,
	torch_dtype=torch.float16
	)

	pipe = CogVideoXImageToVideoPipeline.from_pretrained(
	<span class="hljs-string">"THUDM/CogVideoX-5b-I2V"</span>,
	torch_dtype=torch.bfloat16
	)
	<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="text-to-video" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#text-to-video"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Text-to-Video</span></h2> <p data-svelte-h="svelte-58kgp8">For text-to-video, pass a text prompt. By default, CogVideoX generates a 720x480 video for the best results.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> torch
	<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> CogVideoXPipeline
	<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video

	prompt = <span class="hljs-string">"An elderly gentleman, with a serene expression, sits at the water's edge, a steaming cup of tea by his side. He is engrossed in his artwork, brush in hand, as he renders an oil painting on a canvas that's propped up against a small, weathered table. The sea breeze whispers through his silver hair, gently billowing his loose-fitting white shirt, while the salty air adds an intangible element to his masterpiece in progress. The scene is one of tranquility and inspiration, with the artist's canvas capturing the vibrant hues of the setting sun reflecting off the tranquil sea."</span>

	pipe = CogVideoXPipeline.from_pretrained(
	<span class="hljs-string">"THUDM/CogVideoX-5b"</span>,
	torch_dtype=torch.bfloat16
	)

	pipe.enable_model_cpu_offload()
	pipe.vae.enable_tiling()

	video = pipe(
	prompt=prompt,
	num_videos_per_prompt=<span class="hljs-number">1</span>,
	num_inference_steps=<span class="hljs-number">50</span>,
	num_frames=<span class="hljs-number">49</span>,
	guidance_scale=<span class="hljs-number">6</span>,
	generator=torch.Generator(device=<span class="hljs-string">"cuda"</span>).manual_seed(<span class="hljs-number">42</span>),
	).frames[<span class="hljs-number">0</span>]

	export_to_video(video, <span class="hljs-string">"output.mp4"</span>, fps=<span class="hljs-number">8</span>)
	<!-- HTML_TAG_END --></pre></div> <div class="flex justify-center" data-svelte-h="svelte-ofbxl8"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cogvideox/cogvideox_out.gif" alt="generated image of an astronaut in a jungle"></div> <h2 class="relative group"><a id="image-to-video" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#image-to-video"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Image-to-Video</span></h2> <p data-svelte-h="svelte-1wo8va">You’ll use the <a href="https://huggingface.co/THUDM/CogVideoX-5b-I2V" rel="nofollow">THUDM/CogVideoX-5b-I2V</a> checkpoint for this guide.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> torch
	<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> CogVideoXImageToVideoPipeline
	<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_image

	prompt = <span class="hljs-string">"A vast, shimmering ocean flows gracefully under a twilight sky, its waves undulating in a mesmerizing dance of blues and greens. The surface glints with the last rays of the setting sun, casting golden highlights that ripple across the water. Seagulls soar above, their cries blending with the gentle roar of the waves. The horizon stretches infinitely, where the ocean meets the sky in a seamless blend of hues. Close-ups reveal the intricate patterns of the waves, capturing the fluidity and dynamic beauty of the sea in motion."</span>
	image = load_image(image=<span class="hljs-string">"cogvideox_rocket.png"</span>)
	pipe = CogVideoXImageToVideoPipeline.from_pretrained(
	<span class="hljs-string">"THUDM/CogVideoX-5b-I2V"</span>,
	torch_dtype=torch.bfloat16
	)

	pipe.vae.enable_tiling()
	pipe.vae.enable_slicing()

	video = pipe(
	prompt=prompt,
	image=image,
	num_videos_per_prompt=<span class="hljs-number">1</span>,
	num_inference_steps=<span class="hljs-number">50</span>,
	num_frames=<span class="hljs-number">49</span>,
	guidance_scale=<span class="hljs-number">6</span>,
	generator=torch.Generator(device=<span class="hljs-string">"cuda"</span>).manual_seed(<span class="hljs-number">42</span>),
	).frames[<span class="hljs-number">0</span>]

	export_to_video(video, <span class="hljs-string">"output.mp4"</span>, fps=<span class="hljs-number">8</span>)<!-- HTML_TAG_END --></pre></div> <div class="flex gap-4" data-svelte-h="svelte-1u9aiw3"><div><img class="rounded-xl" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cogvideox/cogvideox_rocket.png"> <figcaption class="mt-2 text-center text-sm text-gray-500">initial image</figcaption></div> <div><img class="rounded-xl" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cogvideox/cogvideox_outrocket.gif"> <figcaption class="mt-2 text-center text-sm text-gray-500">generated video</figcaption></div></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/using-diffusers/cogvideox.md" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span data-svelte-h="svelte-x0xyl0">></span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_inhvqu = {
	assets: "/docs/diffusers/pr_10567/en",
	base: "/docs/diffusers/pr_10567/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/diffusers/pr_10567/en/_app/immutable/entry/start.5ab964f0.js"),
	import("/docs/diffusers/pr_10567/en/_app/immutable/entry/app.d83dbfce.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 234],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 17.4 kB
Xet hash:: aff3b45de91d0ee2a826383cfb9793ef2115ba4854aea3692a28c90b0c4f01e5

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.