Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"PyTorch 2.0","local":"pytorch-20","sections":[{"title":"Scaled dot product attention","local":"scaled-dot-product-attention","sections":[],"depth":2},{"title":"torch.compile","local":"torchcompile","sections":[{"title":"Regional compilation","local":"regional-compilation","sections":[],"depth":3}],"depth":2},{"title":"Benchmark","local":"benchmark","sections":[{"title":"Stable Diffusion text-to-image","local":"stable-diffusion-text-to-image","sections":[],"depth":3},{"title":"Stable Diffusion image-to-image","local":"stable-diffusion-image-to-image","sections":[],"depth":3},{"title":"Stable Diffusion inpainting","local":"stable-diffusion-inpainting","sections":[],"depth":3},{"title":"ControlNet","local":"controlnet","sections":[],"depth":3},{"title":"DeepFloyd IF text-to-image + upscaling","local":"deepfloyd-if-text-to-image--upscaling","sections":[],"depth":3},{"title":"A100 (batch size: 1)","local":"a100-batch-size-1","sections":[],"depth":3},{"title":"A100 (batch size: 4)","local":"a100-batch-size-4","sections":[],"depth":3},{"title":"A100 (batch size: 16)","local":"a100-batch-size-16","sections":[],"depth":3},{"title":"V100 (batch size: 1)","local":"v100-batch-size-1","sections":[],"depth":3},{"title":"V100 (batch size: 4)","local":"v100-batch-size-4","sections":[],"depth":3},{"title":"V100 (batch size: 16)","local":"v100-batch-size-16","sections":[],"depth":3},{"title":"T4 (batch size: 1)","local":"t4-batch-size-1","sections":[],"depth":3},{"title":"T4 (batch size: 4)","local":"t4-batch-size-4","sections":[],"depth":3},{"title":"T4 (batch size: 16)","local":"t4-batch-size-16","sections":[],"depth":3},{"title":"RTX 3090 (batch size: 1)","local":"rtx-3090-batch-size-1","sections":[],"depth":3},{"title":"RTX 3090 (batch size: 4)","local":"rtx-3090-batch-size-4","sections":[],"depth":3},{"title":"RTX 3090 (batch size: 16)","local":"rtx-3090-batch-size-16","sections":[],"depth":3},{"title":"RTX 4090 (batch size: 1)","local":"rtx-4090-batch-size-1","sections":[],"depth":3},{"title":"RTX 4090 (batch size: 4)","local":"rtx-4090-batch-size-4","sections":[],"depth":3},{"title":"RTX 4090 (batch size: 16)","local":"rtx-4090-batch-size-16","sections":[],"depth":3}],"depth":2},{"title":"Notes","local":"notes","sections":[],"depth":2}],"depth":1}"> | |
| <link href="/docs/diffusers/pr_11450/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11450/en/_app/immutable/entry/start.4ce0a7ff.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11450/en/_app/immutable/chunks/scheduler.8c3d61f6.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11450/en/_app/immutable/chunks/singletons.0dcbd085.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11450/en/_app/immutable/chunks/index.0997d446.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11450/en/_app/immutable/chunks/paths.7e11858f.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11450/en/_app/immutable/entry/app.71328ef6.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11450/en/_app/immutable/chunks/index.da70eac4.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11450/en/_app/immutable/nodes/0.cf9c1ba0.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11450/en/_app/immutable/chunks/each.e59479a4.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11450/en/_app/immutable/nodes/232.1e1fa335.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11450/en/_app/immutable/chunks/Tip.1d9b8c37.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11450/en/_app/immutable/chunks/CodeBlock.a9c4becf.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11450/en/_app/immutable/chunks/getInferenceSnippets.07305897.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"PyTorch 2.0","local":"pytorch-20","sections":[{"title":"Scaled dot product attention","local":"scaled-dot-product-attention","sections":[],"depth":2},{"title":"torch.compile","local":"torchcompile","sections":[{"title":"Regional compilation","local":"regional-compilation","sections":[],"depth":3}],"depth":2},{"title":"Benchmark","local":"benchmark","sections":[{"title":"Stable Diffusion text-to-image","local":"stable-diffusion-text-to-image","sections":[],"depth":3},{"title":"Stable Diffusion image-to-image","local":"stable-diffusion-image-to-image","sections":[],"depth":3},{"title":"Stable Diffusion inpainting","local":"stable-diffusion-inpainting","sections":[],"depth":3},{"title":"ControlNet","local":"controlnet","sections":[],"depth":3},{"title":"DeepFloyd IF text-to-image + upscaling","local":"deepfloyd-if-text-to-image--upscaling","sections":[],"depth":3},{"title":"A100 (batch size: 1)","local":"a100-batch-size-1","sections":[],"depth":3},{"title":"A100 (batch size: 4)","local":"a100-batch-size-4","sections":[],"depth":3},{"title":"A100 (batch size: 16)","local":"a100-batch-size-16","sections":[],"depth":3},{"title":"V100 (batch size: 1)","local":"v100-batch-size-1","sections":[],"depth":3},{"title":"V100 (batch size: 4)","local":"v100-batch-size-4","sections":[],"depth":3},{"title":"V100 (batch size: 16)","local":"v100-batch-size-16","sections":[],"depth":3},{"title":"T4 (batch size: 1)","local":"t4-batch-size-1","sections":[],"depth":3},{"title":"T4 (batch size: 4)","local":"t4-batch-size-4","sections":[],"depth":3},{"title":"T4 (batch size: 16)","local":"t4-batch-size-16","sections":[],"depth":3},{"title":"RTX 3090 (batch size: 1)","local":"rtx-3090-batch-size-1","sections":[],"depth":3},{"title":"RTX 3090 (batch size: 4)","local":"rtx-3090-batch-size-4","sections":[],"depth":3},{"title":"RTX 3090 (batch size: 16)","local":"rtx-3090-batch-size-16","sections":[],"depth":3},{"title":"RTX 4090 (batch size: 1)","local":"rtx-4090-batch-size-1","sections":[],"depth":3},{"title":"RTX 4090 (batch size: 4)","local":"rtx-4090-batch-size-4","sections":[],"depth":3},{"title":"RTX 4090 (batch size: 16)","local":"rtx-4090-batch-size-16","sections":[],"depth":3}],"depth":2},{"title":"Notes","local":"notes","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="pytorch-20" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#pytorch-20"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>PyTorch 2.0</span></h1> <p data-svelte-h="svelte-15wmwhw">🤗 Diffusers supports the latest optimizations from <a href="https://pytorch.org/get-started/pytorch-2.0/" rel="nofollow">PyTorch 2.0</a> which include:</p> <ol data-svelte-h="svelte-19p82dc"><li>A memory-efficient attention implementation, scaled dot product attention, without requiring any extra dependencies such as xFormers.</li> <li><a href="https://pytorch.org/tutorials/intermediate/torch_compile_tutorial.html" rel="nofollow"><code>torch.compile</code></a>, a just-in-time (JIT) compiler to provide an extra performance boost when individual models are compiled.</li></ol> <p data-svelte-h="svelte-1u908i1">Both of these optimizations require PyTorch 2.0 or later and 🤗 Diffusers > 0.13.0.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->pip install --upgrade torch diffusers<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="scaled-dot-product-attention" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#scaled-dot-product-attention"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Scaled dot product attention</span></h2> <p data-svelte-h="svelte-1bb69gh"><a href="https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention" rel="nofollow"><code>torch.nn.functional.scaled_dot_product_attention</code></a> (SDPA) is an optimized and memory-efficient attention (similar to xFormers) that automatically enables several other optimizations depending on the model inputs and GPU type. SDPA is enabled by default if you’re using PyTorch 2.0 and the latest version of 🤗 Diffusers, so you don’t need to add anything to your code.</p> <p data-svelte-h="svelte-hvah8d">However, if you want to explicitly enable it, you can set a <a href="/docs/diffusers/pr_11450/en/api/pipelines/overview#diffusers.DiffusionPipeline">DiffusionPipeline</a> to use <a href="/docs/diffusers/pr_11450/en/api/attnprocessor#diffusers.models.attention_processor.AttnProcessor2_0">AttnProcessor2_0</a>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --> import torch | |
| from diffusers import DiffusionPipeline | |
| <span class="hljs-addition">+ from diffusers.models.attention_processor import AttnProcessor2_0</span> | |
| pipe = DiffusionPipeline.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5", torch_dtype=torch.float16, use_safetensors=True).to("cuda") | |
| <span class="hljs-addition">+ pipe.unet.set_attn_processor(AttnProcessor2_0())</span> | |
| prompt = "a photo of an astronaut riding a horse on mars" | |
| image = pipe(prompt).images[0]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-psbnvm">SDPA should be as fast and memory efficient as <code>xFormers</code>; check the <a href="#benchmark">benchmark</a> for more details.</p> <p data-svelte-h="svelte-frsflx">In some cases - such as making the pipeline more deterministic or converting it to other formats - it may be helpful to use the vanilla attention processor, <a href="/docs/diffusers/pr_11450/en/api/attnprocessor#diffusers.models.attention_processor.AttnProcessor">AttnProcessor</a>. To revert to <a href="/docs/diffusers/pr_11450/en/api/attnprocessor#diffusers.models.attention_processor.AttnProcessor">AttnProcessor</a>, call the <a href="/docs/diffusers/pr_11450/en/api/models/unet2d-cond#diffusers.UNet2DConditionModel.set_default_attn_processor">set_default_attn_processor()</a> function on the pipeline:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --> import torch | |
| from diffusers import DiffusionPipeline | |
| pipe = DiffusionPipeline.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5", torch_dtype=torch.float16, use_safetensors=True).to("cuda") | |
| <span class="hljs-addition">+ pipe.unet.set_default_attn_processor()</span> | |
| prompt = "a photo of an astronaut riding a horse on mars" | |
| image = pipe(prompt).images[0]<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="torchcompile" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#torchcompile"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>torch.compile</span></h2> <p data-svelte-h="svelte-1vjh5md">The <code>torch.compile</code> function can often provide an additional speed-up to your PyTorch code. In 🤗 Diffusers, it is usually best to wrap the UNet with <code>torch.compile</code> because it does most of the heavy lifting in the pipeline.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline | |
| <span class="hljs-keyword">import</span> torch | |
| pipe = DiffusionPipeline.from_pretrained(<span class="hljs-string">"stable-diffusion-v1-5/stable-diffusion-v1-5"</span>, torch_dtype=torch.float16, use_safetensors=<span class="hljs-literal">True</span>).to(<span class="hljs-string">"cuda"</span>) | |
| pipe.unet = torch.<span class="hljs-built_in">compile</span>(pipe.unet, mode=<span class="hljs-string">"reduce-overhead"</span>, fullgraph=<span class="hljs-literal">True</span>) | |
| images = pipe(prompt, num_inference_steps=steps, num_images_per_prompt=batch_size).images[<span class="hljs-number">0</span>]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-mm2mnz">Depending on GPU type, <code>torch.compile</code> can provide an <em>additional speed-up</em> of <strong>5-300x</strong> on top of SDPA! If you’re using more recent GPU architectures such as Ampere (A100, 3090), Ada (4090), and Hopper (H100), <code>torch.compile</code> is able to squeeze even more performance out of these GPUs.</p> <p data-svelte-h="svelte-19iy5vt">Compilation requires some time to complete, so it is best suited for situations where you prepare your pipeline once and then perform the same type of inference operations multiple times. For example, calling the compiled pipeline on a different image size triggers compilation again which can be expensive.</p> <p data-svelte-h="svelte-yz7oqo">For more information and different options about <code>torch.compile</code>, refer to the <a href="https://pytorch.org/tutorials/intermediate/torch_compile_tutorial.html" rel="nofollow"><code>torch_compile</code></a> tutorial.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-5loj68">Learn more about other ways PyTorch 2.0 can help optimize your model in the <a href="../tutorials/fast_diffusion">Accelerate inference of text-to-image diffusion models</a> tutorial.</p></div> <h3 class="relative group"><a id="regional-compilation" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#regional-compilation"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Regional compilation</span></h3> <p data-svelte-h="svelte-zu32q8">Compiling the whole model usually has a big problem space for optimization. Models are often composed of multiple repeated blocks. <a href="https://pytorch.org/tutorials/recipes/regional_compilation.html" rel="nofollow">Regional compilation</a> compiles the repeated block first (a transformer encoder block, for example), so that the Torch compiler would re-use its cached/optimized generated code for the other blocks, reducing (often massively) the cold start compilation time observed on the first inference call.</p> <p data-svelte-h="svelte-15aywvs">Enabling regional compilation might require simple yet intrusive changes to the | |
| modeling code. However, 🤗 Accelerate provides a utility <a href="https://huggingface.co/docs/accelerate/main/en/usage_guides/compilation#how-to-use-regional-compilation" rel="nofollow"><code>compile_regions()</code></a> which automatically compiles | |
| the repeated blocks of the provided <code>nn.Module</code> sequentially, and the rest of the model separately. This helps with reducing cold start time while keeping most (if not all) of the speedup you would get from full compilation.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># Make sure you're on the latest `accelerate`: `pip install -U accelerate`.</span> | |
| <span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> compile_regions | |
| pipe.unet = compile_regions(pipe.unet, mode=<span class="hljs-string">"reduce-overhead"</span>, fullgraph=<span class="hljs-literal">True</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-q3cpu5">As you may have noticed <code>compile_regions()</code> takes the same arguments as <code>torch.compile()</code>, allowing flexibility.</p> <h2 class="relative group"><a id="benchmark" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#benchmark"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Benchmark</span></h2> <p data-svelte-h="svelte-km3nc0">We conducted a comprehensive benchmark with PyTorch 2.0’s efficient attention implementation and <code>torch.compile</code> across different GPUs and batch sizes for five of our most used pipelines. The code is benchmarked on 🤗 Diffusers v0.17.0.dev0 to optimize <code>torch.compile</code> usage (see <a href="https://github.com/huggingface/diffusers/pull/3313" rel="nofollow">here</a> for more details).</p> <p data-svelte-h="svelte-pe5bn7">Expand the dropdown below to find the code used to benchmark each pipeline:</p> <details> <h3 class="relative group"><a id="stable-diffusion-text-to-image" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#stable-diffusion-text-to-image"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Stable Diffusion text-to-image</span></h3> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline | |
| <span class="hljs-keyword">import</span> torch | |
| path = <span class="hljs-string">"stable-diffusion-v1-5/stable-diffusion-v1-5"</span> | |
| run_compile = <span class="hljs-literal">True</span> <span class="hljs-comment"># Set True / False</span> | |
| pipe = DiffusionPipeline.from_pretrained(path, torch_dtype=torch.float16, use_safetensors=<span class="hljs-literal">True</span>) | |
| pipe = pipe.to(<span class="hljs-string">"cuda"</span>) | |
| pipe.unet.to(memory_format=torch.channels_last) | |
| <span class="hljs-keyword">if</span> run_compile: | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">"Run torch compile"</span>) | |
| pipe.unet = torch.<span class="hljs-built_in">compile</span>(pipe.unet, mode=<span class="hljs-string">"reduce-overhead"</span>, fullgraph=<span class="hljs-literal">True</span>) | |
| prompt = <span class="hljs-string">"ghibli style, a fantasy landscape with castles"</span> | |
| <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">3</span>): | |
| images = pipe(prompt=prompt).images<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="stable-diffusion-image-to-image" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#stable-diffusion-image-to-image"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Stable Diffusion image-to-image</span></h3> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionImg2ImgPipeline | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image | |
| <span class="hljs-keyword">import</span> torch | |
| url = <span class="hljs-string">"https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg"</span> | |
| init_image = load_image(url) | |
| init_image = init_image.resize((<span class="hljs-number">512</span>, <span class="hljs-number">512</span>)) | |
| path = <span class="hljs-string">"stable-diffusion-v1-5/stable-diffusion-v1-5"</span> | |
| run_compile = <span class="hljs-literal">True</span> <span class="hljs-comment"># Set True / False</span> | |
| pipe = StableDiffusionImg2ImgPipeline.from_pretrained(path, torch_dtype=torch.float16, use_safetensors=<span class="hljs-literal">True</span>) | |
| pipe = pipe.to(<span class="hljs-string">"cuda"</span>) | |
| pipe.unet.to(memory_format=torch.channels_last) | |
| <span class="hljs-keyword">if</span> run_compile: | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">"Run torch compile"</span>) | |
| pipe.unet = torch.<span class="hljs-built_in">compile</span>(pipe.unet, mode=<span class="hljs-string">"reduce-overhead"</span>, fullgraph=<span class="hljs-literal">True</span>) | |
| prompt = <span class="hljs-string">"ghibli style, a fantasy landscape with castles"</span> | |
| <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">3</span>): | |
| image = pipe(prompt=prompt, image=init_image).images[<span class="hljs-number">0</span>]<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="stable-diffusion-inpainting" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#stable-diffusion-inpainting"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Stable Diffusion inpainting</span></h3> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionInpaintPipeline | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image | |
| <span class="hljs-keyword">import</span> torch | |
| img_url = <span class="hljs-string">"https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png"</span> | |
| mask_url = <span class="hljs-string">"https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png"</span> | |
| init_image = load_image(img_url).resize((<span class="hljs-number">512</span>, <span class="hljs-number">512</span>)) | |
| mask_image = load_image(mask_url).resize((<span class="hljs-number">512</span>, <span class="hljs-number">512</span>)) | |
| path = <span class="hljs-string">"runwayml/stable-diffusion-inpainting"</span> | |
| run_compile = <span class="hljs-literal">True</span> <span class="hljs-comment"># Set True / False</span> | |
| pipe = StableDiffusionInpaintPipeline.from_pretrained(path, torch_dtype=torch.float16, use_safetensors=<span class="hljs-literal">True</span>) | |
| pipe = pipe.to(<span class="hljs-string">"cuda"</span>) | |
| pipe.unet.to(memory_format=torch.channels_last) | |
| <span class="hljs-keyword">if</span> run_compile: | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">"Run torch compile"</span>) | |
| pipe.unet = torch.<span class="hljs-built_in">compile</span>(pipe.unet, mode=<span class="hljs-string">"reduce-overhead"</span>, fullgraph=<span class="hljs-literal">True</span>) | |
| prompt = <span class="hljs-string">"ghibli style, a fantasy landscape with castles"</span> | |
| <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">3</span>): | |
| image = pipe(prompt=prompt, image=init_image, mask_image=mask_image).images[<span class="hljs-number">0</span>]<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="controlnet" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#controlnet"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>ControlNet</span></h3> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionControlNetPipeline, ControlNetModel | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image | |
| <span class="hljs-keyword">import</span> torch | |
| url = <span class="hljs-string">"https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg"</span> | |
| init_image = load_image(url) | |
| init_image = init_image.resize((<span class="hljs-number">512</span>, <span class="hljs-number">512</span>)) | |
| path = <span class="hljs-string">"stable-diffusion-v1-5/stable-diffusion-v1-5"</span> | |
| run_compile = <span class="hljs-literal">True</span> <span class="hljs-comment"># Set True / False</span> | |
| controlnet = ControlNetModel.from_pretrained(<span class="hljs-string">"lllyasviel/sd-controlnet-canny"</span>, torch_dtype=torch.float16, use_safetensors=<span class="hljs-literal">True</span>) | |
| pipe = StableDiffusionControlNetPipeline.from_pretrained( | |
| path, controlnet=controlnet, torch_dtype=torch.float16, use_safetensors=<span class="hljs-literal">True</span> | |
| ) | |
| pipe = pipe.to(<span class="hljs-string">"cuda"</span>) | |
| pipe.unet.to(memory_format=torch.channels_last) | |
| pipe.controlnet.to(memory_format=torch.channels_last) | |
| <span class="hljs-keyword">if</span> run_compile: | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">"Run torch compile"</span>) | |
| pipe.unet = torch.<span class="hljs-built_in">compile</span>(pipe.unet, mode=<span class="hljs-string">"reduce-overhead"</span>, fullgraph=<span class="hljs-literal">True</span>) | |
| pipe.controlnet = torch.<span class="hljs-built_in">compile</span>(pipe.controlnet, mode=<span class="hljs-string">"reduce-overhead"</span>, fullgraph=<span class="hljs-literal">True</span>) | |
| prompt = <span class="hljs-string">"ghibli style, a fantasy landscape with castles"</span> | |
| <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">3</span>): | |
| image = pipe(prompt=prompt, image=init_image).images[<span class="hljs-number">0</span>]<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="deepfloyd-if-text-to-image--upscaling" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#deepfloyd-if-text-to-image--upscaling"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>DeepFloyd IF text-to-image + upscaling</span></h3> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline | |
| <span class="hljs-keyword">import</span> torch | |
| run_compile = <span class="hljs-literal">True</span> <span class="hljs-comment"># Set True / False</span> | |
| pipe_1 = DiffusionPipeline.from_pretrained(<span class="hljs-string">"DeepFloyd/IF-I-M-v1.0"</span>, variant=<span class="hljs-string">"fp16"</span>, text_encoder=<span class="hljs-literal">None</span>, torch_dtype=torch.float16, use_safetensors=<span class="hljs-literal">True</span>) | |
| pipe_1.to(<span class="hljs-string">"cuda"</span>) | |
| pipe_2 = DiffusionPipeline.from_pretrained(<span class="hljs-string">"DeepFloyd/IF-II-M-v1.0"</span>, variant=<span class="hljs-string">"fp16"</span>, text_encoder=<span class="hljs-literal">None</span>, torch_dtype=torch.float16, use_safetensors=<span class="hljs-literal">True</span>) | |
| pipe_2.to(<span class="hljs-string">"cuda"</span>) | |
| pipe_3 = DiffusionPipeline.from_pretrained(<span class="hljs-string">"stabilityai/stable-diffusion-x4-upscaler"</span>, torch_dtype=torch.float16, use_safetensors=<span class="hljs-literal">True</span>) | |
| pipe_3.to(<span class="hljs-string">"cuda"</span>) | |
| pipe_1.unet.to(memory_format=torch.channels_last) | |
| pipe_2.unet.to(memory_format=torch.channels_last) | |
| pipe_3.unet.to(memory_format=torch.channels_last) | |
| <span class="hljs-keyword">if</span> run_compile: | |
| pipe_1.unet = torch.<span class="hljs-built_in">compile</span>(pipe_1.unet, mode=<span class="hljs-string">"reduce-overhead"</span>, fullgraph=<span class="hljs-literal">True</span>) | |
| pipe_2.unet = torch.<span class="hljs-built_in">compile</span>(pipe_2.unet, mode=<span class="hljs-string">"reduce-overhead"</span>, fullgraph=<span class="hljs-literal">True</span>) | |
| pipe_3.unet = torch.<span class="hljs-built_in">compile</span>(pipe_3.unet, mode=<span class="hljs-string">"reduce-overhead"</span>, fullgraph=<span class="hljs-literal">True</span>) | |
| prompt = <span class="hljs-string">"the blue hulk"</span> | |
| prompt_embeds = torch.randn((<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">4096</span>), dtype=torch.float16) | |
| neg_prompt_embeds = torch.randn((<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">4096</span>), dtype=torch.float16) | |
| <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">3</span>): | |
| image_1 = pipe_1(prompt_embeds=prompt_embeds, negative_prompt_embeds=neg_prompt_embeds, output_type=<span class="hljs-string">"pt"</span>).images | |
| image_2 = pipe_2(image=image_1, prompt_embeds=prompt_embeds, negative_prompt_embeds=neg_prompt_embeds, output_type=<span class="hljs-string">"pt"</span>).images | |
| image_3 = pipe_3(prompt=prompt, image=image_1, noise_level=<span class="hljs-number">100</span>).images<!-- HTML_TAG_END --></pre></div></details> <p data-svelte-h="svelte-1vu9ie">The graph below highlights the relative speed-ups for the <a href="/docs/diffusers/pr_11450/en/api/pipelines/stable_diffusion/text2img#diffusers.StableDiffusionPipeline">StableDiffusionPipeline</a> across five GPU families with PyTorch 2.0 and <code>torch.compile</code> enabled. The benchmarks for the following graphs are measured in <em>number of iterations/second</em>.</p> <p data-svelte-h="svelte-1sjwyaz"><img src="https://huggingface.co/datasets/diffusers/docs-images/resolve/main/pt2_benchmarks/t2i_speedup.png" alt="t2i_speedup"></p> <p data-svelte-h="svelte-1zzsz7">To give you an even better idea of how this speed-up holds for the other pipelines, consider the following | |
| graph for an A100 with PyTorch 2.0 and <code>torch.compile</code>:</p> <p data-svelte-h="svelte-19p9vzv"><img src="https://huggingface.co/datasets/diffusers/docs-images/resolve/main/pt2_benchmarks/a100_numbers.png" alt="a100_numbers"></p> <p data-svelte-h="svelte-18iwyus">In the following tables, we report our findings in terms of the <em>number of iterations/second</em>.</p> <h3 class="relative group"><a id="a100-batch-size-1" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#a100-batch-size-1"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>A100 (batch size: 1)</span></h3> <table data-svelte-h="svelte-14f48w"><thead><tr><th align="center"><strong>Pipeline</strong></th> <th align="center"><strong>torch 2.0 - <br>no compile</strong></th> <th align="center"><strong>torch nightly - <br>no compile</strong></th> <th align="center"><strong>torch 2.0 - <br>compile</strong></th> <th align="center"><strong>torch nightly - <br>compile</strong></th></tr></thead> <tbody><tr><td align="center">SD - txt2img</td> <td align="center">21.66</td> <td align="center">23.13</td> <td align="center">44.03</td> <td align="center">49.74</td></tr> <tr><td align="center">SD - img2img</td> <td align="center">21.81</td> <td align="center">22.40</td> <td align="center">43.92</td> <td align="center">46.32</td></tr> <tr><td align="center">SD - inpaint</td> <td align="center">22.24</td> <td align="center">23.23</td> <td align="center">43.76</td> <td align="center">49.25</td></tr> <tr><td align="center">SD - controlnet</td> <td align="center">15.02</td> <td align="center">15.82</td> <td align="center">32.13</td> <td align="center">36.08</td></tr> <tr><td align="center">IF</td> <td align="center">20.21 / <br>13.84 / <br>24.00</td> <td align="center">20.12 / <br>13.70 / <br>24.03</td> <td align="center">❌</td> <td align="center">97.34 / <br>27.23 / <br>111.66</td></tr> <tr><td align="center">SDXL - txt2img</td> <td align="center">8.64</td> <td align="center">9.9</td> <td align="center">-</td> <td align="center">-</td></tr></tbody></table> <h3 class="relative group"><a id="a100-batch-size-4" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#a100-batch-size-4"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>A100 (batch size: 4)</span></h3> <table data-svelte-h="svelte-19bbd89"><thead><tr><th align="center"><strong>Pipeline</strong></th> <th align="center"><strong>torch 2.0 - <br>no compile</strong></th> <th align="center"><strong>torch nightly - <br>no compile</strong></th> <th align="center"><strong>torch 2.0 - <br>compile</strong></th> <th align="center"><strong>torch nightly - <br>compile</strong></th></tr></thead> <tbody><tr><td align="center">SD - txt2img</td> <td align="center">11.6</td> <td align="center">13.12</td> <td align="center">14.62</td> <td align="center">17.27</td></tr> <tr><td align="center">SD - img2img</td> <td align="center">11.47</td> <td align="center">13.06</td> <td align="center">14.66</td> <td align="center">17.25</td></tr> <tr><td align="center">SD - inpaint</td> <td align="center">11.67</td> <td align="center">13.31</td> <td align="center">14.88</td> <td align="center">17.48</td></tr> <tr><td align="center">SD - controlnet</td> <td align="center">8.28</td> <td align="center">9.38</td> <td align="center">10.51</td> <td align="center">12.41</td></tr> <tr><td align="center">IF</td> <td align="center">25.02</td> <td align="center">18.04</td> <td align="center">❌</td> <td align="center">48.47</td></tr> <tr><td align="center">SDXL - txt2img</td> <td align="center">2.44</td> <td align="center">2.74</td> <td align="center">-</td> <td align="center">-</td></tr></tbody></table> <h3 class="relative group"><a id="a100-batch-size-16" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#a100-batch-size-16"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>A100 (batch size: 16)</span></h3> <table data-svelte-h="svelte-cd9a2o"><thead><tr><th align="center"><strong>Pipeline</strong></th> <th align="center"><strong>torch 2.0 - <br>no compile</strong></th> <th align="center"><strong>torch nightly - <br>no compile</strong></th> <th align="center"><strong>torch 2.0 - <br>compile</strong></th> <th align="center"><strong>torch nightly - <br>compile</strong></th></tr></thead> <tbody><tr><td align="center">SD - txt2img</td> <td align="center">3.04</td> <td align="center">3.6</td> <td align="center">3.83</td> <td align="center">4.68</td></tr> <tr><td align="center">SD - img2img</td> <td align="center">2.98</td> <td align="center">3.58</td> <td align="center">3.83</td> <td align="center">4.67</td></tr> <tr><td align="center">SD - inpaint</td> <td align="center">3.04</td> <td align="center">3.66</td> <td align="center">3.9</td> <td align="center">4.76</td></tr> <tr><td align="center">SD - controlnet</td> <td align="center">2.15</td> <td align="center">2.58</td> <td align="center">2.74</td> <td align="center">3.35</td></tr> <tr><td align="center">IF</td> <td align="center">8.78</td> <td align="center">9.82</td> <td align="center">❌</td> <td align="center">16.77</td></tr> <tr><td align="center">SDXL - txt2img</td> <td align="center">0.64</td> <td align="center">0.72</td> <td align="center">-</td> <td align="center">-</td></tr></tbody></table> <h3 class="relative group"><a id="v100-batch-size-1" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#v100-batch-size-1"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>V100 (batch size: 1)</span></h3> <table data-svelte-h="svelte-9j8rj7"><thead><tr><th align="center"><strong>Pipeline</strong></th> <th align="center"><strong>torch 2.0 - <br>no compile</strong></th> <th align="center"><strong>torch nightly - <br>no compile</strong></th> <th align="center"><strong>torch 2.0 - <br>compile</strong></th> <th align="center"><strong>torch nightly - <br>compile</strong></th></tr></thead> <tbody><tr><td align="center">SD - txt2img</td> <td align="center">18.99</td> <td align="center">19.14</td> <td align="center">20.95</td> <td align="center">22.17</td></tr> <tr><td align="center">SD - img2img</td> <td align="center">18.56</td> <td align="center">19.18</td> <td align="center">20.95</td> <td align="center">22.11</td></tr> <tr><td align="center">SD - inpaint</td> <td align="center">19.14</td> <td align="center">19.06</td> <td align="center">21.08</td> <td align="center">22.20</td></tr> <tr><td align="center">SD - controlnet</td> <td align="center">13.48</td> <td align="center">13.93</td> <td align="center">15.18</td> <td align="center">15.88</td></tr> <tr><td align="center">IF</td> <td align="center">20.01 / <br>9.08 / <br>23.34</td> <td align="center">19.79 / <br>8.98 / <br>24.10</td> <td align="center">❌</td> <td align="center">55.75 / <br>11.57 / <br>57.67</td></tr></tbody></table> <h3 class="relative group"><a id="v100-batch-size-4" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#v100-batch-size-4"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>V100 (batch size: 4)</span></h3> <table data-svelte-h="svelte-1xs3uwi"><thead><tr><th align="center"><strong>Pipeline</strong></th> <th align="center"><strong>torch 2.0 - <br>no compile</strong></th> <th align="center"><strong>torch nightly - <br>no compile</strong></th> <th align="center"><strong>torch 2.0 - <br>compile</strong></th> <th align="center"><strong>torch nightly - <br>compile</strong></th></tr></thead> <tbody><tr><td align="center">SD - txt2img</td> <td align="center">5.96</td> <td align="center">5.89</td> <td align="center">6.83</td> <td align="center">6.86</td></tr> <tr><td align="center">SD - img2img</td> <td align="center">5.90</td> <td align="center">5.91</td> <td align="center">6.81</td> <td align="center">6.82</td></tr> <tr><td align="center">SD - inpaint</td> <td align="center">5.99</td> <td align="center">6.03</td> <td align="center">6.93</td> <td align="center">6.95</td></tr> <tr><td align="center">SD - controlnet</td> <td align="center">4.26</td> <td align="center">4.29</td> <td align="center">4.92</td> <td align="center">4.93</td></tr> <tr><td align="center">IF</td> <td align="center">15.41</td> <td align="center">14.76</td> <td align="center">❌</td> <td align="center">22.95</td></tr></tbody></table> <h3 class="relative group"><a id="v100-batch-size-16" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#v100-batch-size-16"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>V100 (batch size: 16)</span></h3> <table data-svelte-h="svelte-1elkjwp"><thead><tr><th align="center"><strong>Pipeline</strong></th> <th align="center"><strong>torch 2.0 - <br>no compile</strong></th> <th align="center"><strong>torch nightly - <br>no compile</strong></th> <th align="center"><strong>torch 2.0 - <br>compile</strong></th> <th align="center"><strong>torch nightly - <br>compile</strong></th></tr></thead> <tbody><tr><td align="center">SD - txt2img</td> <td align="center">1.66</td> <td align="center">1.66</td> <td align="center">1.92</td> <td align="center">1.90</td></tr> <tr><td align="center">SD - img2img</td> <td align="center">1.65</td> <td align="center">1.65</td> <td align="center">1.91</td> <td align="center">1.89</td></tr> <tr><td align="center">SD - inpaint</td> <td align="center">1.69</td> <td align="center">1.69</td> <td align="center">1.95</td> <td align="center">1.93</td></tr> <tr><td align="center">SD - controlnet</td> <td align="center">1.19</td> <td align="center">1.19</td> <td align="center">OOM after warmup</td> <td align="center">1.36</td></tr> <tr><td align="center">IF</td> <td align="center">5.43</td> <td align="center">5.29</td> <td align="center">❌</td> <td align="center">7.06</td></tr></tbody></table> <h3 class="relative group"><a id="t4-batch-size-1" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#t4-batch-size-1"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>T4 (batch size: 1)</span></h3> <table data-svelte-h="svelte-sqqgka"><thead><tr><th align="center"><strong>Pipeline</strong></th> <th align="center"><strong>torch 2.0 - <br>no compile</strong></th> <th align="center"><strong>torch nightly - <br>no compile</strong></th> <th align="center"><strong>torch 2.0 - <br>compile</strong></th> <th align="center"><strong>torch nightly - <br>compile</strong></th></tr></thead> <tbody><tr><td align="center">SD - txt2img</td> <td align="center">6.9</td> <td align="center">6.95</td> <td align="center">7.3</td> <td align="center">7.56</td></tr> <tr><td align="center">SD - img2img</td> <td align="center">6.84</td> <td align="center">6.99</td> <td align="center">7.04</td> <td align="center">7.55</td></tr> <tr><td align="center">SD - inpaint</td> <td align="center">6.91</td> <td align="center">6.7</td> <td align="center">7.01</td> <td align="center">7.37</td></tr> <tr><td align="center">SD - controlnet</td> <td align="center">4.89</td> <td align="center">4.86</td> <td align="center">5.35</td> <td align="center">5.48</td></tr> <tr><td align="center">IF</td> <td align="center">17.42 / <br>2.47 / <br>18.52</td> <td align="center">16.96 / <br>2.45 / <br>18.69</td> <td align="center">❌</td> <td align="center">24.63 / <br>2.47 / <br>23.39</td></tr> <tr><td align="center">SDXL - txt2img</td> <td align="center">1.15</td> <td align="center">1.16</td> <td align="center">-</td> <td align="center">-</td></tr></tbody></table> <h3 class="relative group"><a id="t4-batch-size-4" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#t4-batch-size-4"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>T4 (batch size: 4)</span></h3> <table data-svelte-h="svelte-15f755w"><thead><tr><th align="center"><strong>Pipeline</strong></th> <th align="center"><strong>torch 2.0 - <br>no compile</strong></th> <th align="center"><strong>torch nightly - <br>no compile</strong></th> <th align="center"><strong>torch 2.0 - <br>compile</strong></th> <th align="center"><strong>torch nightly - <br>compile</strong></th></tr></thead> <tbody><tr><td align="center">SD - txt2img</td> <td align="center">1.79</td> <td align="center">1.79</td> <td align="center">2.03</td> <td align="center">1.99</td></tr> <tr><td align="center">SD - img2img</td> <td align="center">1.77</td> <td align="center">1.77</td> <td align="center">2.05</td> <td align="center">2.04</td></tr> <tr><td align="center">SD - inpaint</td> <td align="center">1.81</td> <td align="center">1.82</td> <td align="center">2.09</td> <td align="center">2.09</td></tr> <tr><td align="center">SD - controlnet</td> <td align="center">1.34</td> <td align="center">1.27</td> <td align="center">1.47</td> <td align="center">1.46</td></tr> <tr><td align="center">IF</td> <td align="center">5.79</td> <td align="center">5.61</td> <td align="center">❌</td> <td align="center">7.39</td></tr> <tr><td align="center">SDXL - txt2img</td> <td align="center">0.288</td> <td align="center">0.289</td> <td align="center">-</td> <td align="center">-</td></tr></tbody></table> <h3 class="relative group"><a id="t4-batch-size-16" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#t4-batch-size-16"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>T4 (batch size: 16)</span></h3> <table data-svelte-h="svelte-uuno4d"><thead><tr><th align="center"><strong>Pipeline</strong></th> <th align="center"><strong>torch 2.0 - <br>no compile</strong></th> <th align="center"><strong>torch nightly - <br>no compile</strong></th> <th align="center"><strong>torch 2.0 - <br>compile</strong></th> <th align="center"><strong>torch nightly - <br>compile</strong></th></tr></thead> <tbody><tr><td align="center">SD - txt2img</td> <td align="center">2.34s</td> <td align="center">2.30s</td> <td align="center">OOM after 2nd iteration</td> <td align="center">1.99s</td></tr> <tr><td align="center">SD - img2img</td> <td align="center">2.35s</td> <td align="center">2.31s</td> <td align="center">OOM after warmup</td> <td align="center">2.00s</td></tr> <tr><td align="center">SD - inpaint</td> <td align="center">2.30s</td> <td align="center">2.26s</td> <td align="center">OOM after 2nd iteration</td> <td align="center">1.95s</td></tr> <tr><td align="center">SD - controlnet</td> <td align="center">OOM after 2nd iteration</td> <td align="center">OOM after 2nd iteration</td> <td align="center">OOM after warmup</td> <td align="center">OOM after warmup</td></tr> <tr><td align="center">IF *</td> <td align="center">1.44</td> <td align="center">1.44</td> <td align="center">❌</td> <td align="center">1.94</td></tr> <tr><td align="center">SDXL - txt2img</td> <td align="center">OOM</td> <td align="center">OOM</td> <td align="center">-</td> <td align="center">-</td></tr></tbody></table> <h3 class="relative group"><a id="rtx-3090-batch-size-1" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#rtx-3090-batch-size-1"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>RTX 3090 (batch size: 1)</span></h3> <table data-svelte-h="svelte-1hbo08l"><thead><tr><th align="center"><strong>Pipeline</strong></th> <th align="center"><strong>torch 2.0 - <br>no compile</strong></th> <th align="center"><strong>torch nightly - <br>no compile</strong></th> <th align="center"><strong>torch 2.0 - <br>compile</strong></th> <th align="center"><strong>torch nightly - <br>compile</strong></th></tr></thead> <tbody><tr><td align="center">SD - txt2img</td> <td align="center">22.56</td> <td align="center">22.84</td> <td align="center">23.84</td> <td align="center">25.69</td></tr> <tr><td align="center">SD - img2img</td> <td align="center">22.25</td> <td align="center">22.61</td> <td align="center">24.1</td> <td align="center">25.83</td></tr> <tr><td align="center">SD - inpaint</td> <td align="center">22.22</td> <td align="center">22.54</td> <td align="center">24.26</td> <td align="center">26.02</td></tr> <tr><td align="center">SD - controlnet</td> <td align="center">16.03</td> <td align="center">16.33</td> <td align="center">17.38</td> <td align="center">18.56</td></tr> <tr><td align="center">IF</td> <td align="center">27.08 / <br>9.07 / <br>31.23</td> <td align="center">26.75 / <br>8.92 / <br>31.47</td> <td align="center">❌</td> <td align="center">68.08 / <br>11.16 / <br>65.29</td></tr></tbody></table> <h3 class="relative group"><a id="rtx-3090-batch-size-4" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#rtx-3090-batch-size-4"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>RTX 3090 (batch size: 4)</span></h3> <table data-svelte-h="svelte-abi9u4"><thead><tr><th align="center"><strong>Pipeline</strong></th> <th align="center"><strong>torch 2.0 - <br>no compile</strong></th> <th align="center"><strong>torch nightly - <br>no compile</strong></th> <th align="center"><strong>torch 2.0 - <br>compile</strong></th> <th align="center"><strong>torch nightly - <br>compile</strong></th></tr></thead> <tbody><tr><td align="center">SD - txt2img</td> <td align="center">6.46</td> <td align="center">6.35</td> <td align="center">7.29</td> <td align="center">7.3</td></tr> <tr><td align="center">SD - img2img</td> <td align="center">6.33</td> <td align="center">6.27</td> <td align="center">7.31</td> <td align="center">7.26</td></tr> <tr><td align="center">SD - inpaint</td> <td align="center">6.47</td> <td align="center">6.4</td> <td align="center">7.44</td> <td align="center">7.39</td></tr> <tr><td align="center">SD - controlnet</td> <td align="center">4.59</td> <td align="center">4.54</td> <td align="center">5.27</td> <td align="center">5.26</td></tr> <tr><td align="center">IF</td> <td align="center">16.81</td> <td align="center">16.62</td> <td align="center">❌</td> <td align="center">21.57</td></tr></tbody></table> <h3 class="relative group"><a id="rtx-3090-batch-size-16" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#rtx-3090-batch-size-16"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>RTX 3090 (batch size: 16)</span></h3> <table data-svelte-h="svelte-11m8i0f"><thead><tr><th align="center"><strong>Pipeline</strong></th> <th align="center"><strong>torch 2.0 - <br>no compile</strong></th> <th align="center"><strong>torch nightly - <br>no compile</strong></th> <th align="center"><strong>torch 2.0 - <br>compile</strong></th> <th align="center"><strong>torch nightly - <br>compile</strong></th></tr></thead> <tbody><tr><td align="center">SD - txt2img</td> <td align="center">1.7</td> <td align="center">1.69</td> <td align="center">1.93</td> <td align="center">1.91</td></tr> <tr><td align="center">SD - img2img</td> <td align="center">1.68</td> <td align="center">1.67</td> <td align="center">1.93</td> <td align="center">1.9</td></tr> <tr><td align="center">SD - inpaint</td> <td align="center">1.72</td> <td align="center">1.71</td> <td align="center">1.97</td> <td align="center">1.94</td></tr> <tr><td align="center">SD - controlnet</td> <td align="center">1.23</td> <td align="center">1.22</td> <td align="center">1.4</td> <td align="center">1.38</td></tr> <tr><td align="center">IF</td> <td align="center">5.01</td> <td align="center">5.00</td> <td align="center">❌</td> <td align="center">6.33</td></tr></tbody></table> <h3 class="relative group"><a id="rtx-4090-batch-size-1" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#rtx-4090-batch-size-1"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>RTX 4090 (batch size: 1)</span></h3> <table data-svelte-h="svelte-1i4s3fg"><thead><tr><th align="center"><strong>Pipeline</strong></th> <th align="center"><strong>torch 2.0 - <br>no compile</strong></th> <th align="center"><strong>torch nightly - <br>no compile</strong></th> <th align="center"><strong>torch 2.0 - <br>compile</strong></th> <th align="center"><strong>torch nightly - <br>compile</strong></th></tr></thead> <tbody><tr><td align="center">SD - txt2img</td> <td align="center">40.5</td> <td align="center">41.89</td> <td align="center">44.65</td> <td align="center">49.81</td></tr> <tr><td align="center">SD - img2img</td> <td align="center">40.39</td> <td align="center">41.95</td> <td align="center">44.46</td> <td align="center">49.8</td></tr> <tr><td align="center">SD - inpaint</td> <td align="center">40.51</td> <td align="center">41.88</td> <td align="center">44.58</td> <td align="center">49.72</td></tr> <tr><td align="center">SD - controlnet</td> <td align="center">29.27</td> <td align="center">30.29</td> <td align="center">32.26</td> <td align="center">36.03</td></tr> <tr><td align="center">IF</td> <td align="center">69.71 / <br>18.78 / <br>85.49</td> <td align="center">69.13 / <br>18.80 / <br>85.56</td> <td align="center">❌</td> <td align="center">124.60 / <br>26.37 / <br>138.79</td></tr> <tr><td align="center">SDXL - txt2img</td> <td align="center">6.8</td> <td align="center">8.18</td> <td align="center">-</td> <td align="center">-</td></tr></tbody></table> <h3 class="relative group"><a id="rtx-4090-batch-size-4" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#rtx-4090-batch-size-4"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>RTX 4090 (batch size: 4)</span></h3> <table data-svelte-h="svelte-13cqeai"><thead><tr><th align="center"><strong>Pipeline</strong></th> <th align="center"><strong>torch 2.0 - <br>no compile</strong></th> <th align="center"><strong>torch nightly - <br>no compile</strong></th> <th align="center"><strong>torch 2.0 - <br>compile</strong></th> <th align="center"><strong>torch nightly - <br>compile</strong></th></tr></thead> <tbody><tr><td align="center">SD - txt2img</td> <td align="center">12.62</td> <td align="center">12.84</td> <td align="center">15.32</td> <td align="center">15.59</td></tr> <tr><td align="center">SD - img2img</td> <td align="center">12.61</td> <td align="center">12,.79</td> <td align="center">15.35</td> <td align="center">15.66</td></tr> <tr><td align="center">SD - inpaint</td> <td align="center">12.65</td> <td align="center">12.81</td> <td align="center">15.3</td> <td align="center">15.58</td></tr> <tr><td align="center">SD - controlnet</td> <td align="center">9.1</td> <td align="center">9.25</td> <td align="center">11.03</td> <td align="center">11.22</td></tr> <tr><td align="center">IF</td> <td align="center">31.88</td> <td align="center">31.14</td> <td align="center">❌</td> <td align="center">43.92</td></tr> <tr><td align="center">SDXL - txt2img</td> <td align="center">2.19</td> <td align="center">2.35</td> <td align="center">-</td> <td align="center">-</td></tr></tbody></table> <h3 class="relative group"><a id="rtx-4090-batch-size-16" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#rtx-4090-batch-size-16"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>RTX 4090 (batch size: 16)</span></h3> <table data-svelte-h="svelte-16al08t"><thead><tr><th align="center"><strong>Pipeline</strong></th> <th align="center"><strong>torch 2.0 - <br>no compile</strong></th> <th align="center"><strong>torch nightly - <br>no compile</strong></th> <th align="center"><strong>torch 2.0 - <br>compile</strong></th> <th align="center"><strong>torch nightly - <br>compile</strong></th></tr></thead> <tbody><tr><td align="center">SD - txt2img</td> <td align="center">3.17</td> <td align="center">3.2</td> <td align="center">3.84</td> <td align="center">3.85</td></tr> <tr><td align="center">SD - img2img</td> <td align="center">3.16</td> <td align="center">3.2</td> <td align="center">3.84</td> <td align="center">3.85</td></tr> <tr><td align="center">SD - inpaint</td> <td align="center">3.17</td> <td align="center">3.2</td> <td align="center">3.85</td> <td align="center">3.85</td></tr> <tr><td align="center">SD - controlnet</td> <td align="center">2.23</td> <td align="center">2.3</td> <td align="center">2.7</td> <td align="center">2.75</td></tr> <tr><td align="center">IF</td> <td align="center">9.26</td> <td align="center">9.2</td> <td align="center">❌</td> <td align="center">13.31</td></tr> <tr><td align="center">SDXL - txt2img</td> <td align="center">0.52</td> <td align="center">0.53</td> <td align="center">-</td> <td align="center">-</td></tr></tbody></table> <h2 class="relative group"><a id="notes" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#notes"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Notes</span></h2> <ul data-svelte-h="svelte-1xg6qk7"><li>Follow this <a href="https://github.com/huggingface/diffusers/pull/3313" rel="nofollow">PR</a> for more details on the environment used for conducting the benchmarks.</li> <li>For the DeepFloyd IF pipeline where batch sizes > 1, we only used a batch size of > 1 in the first IF pipeline for text-to-image generation and NOT for upscaling. That means the two upscaling pipelines received a batch size of 1.</li></ul> <p data-svelte-h="svelte-7zliwv"><em>Thanks to <a href="https://github.com/Chillee" rel="nofollow">Horace He</a> from the PyTorch team for their support in improving our support of <code>torch.compile()</code> in Diffusers.</em></p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/optimization/torch2.0.md" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span data-svelte-h="svelte-x0xyl0">></span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p> | |
| <script> | |
| { | |
| __sveltekit_1tg59h4 = { | |
| assets: "/docs/diffusers/pr_11450/en", | |
| base: "/docs/diffusers/pr_11450/en", | |
| env: {} | |
| }; | |
| const element = document.currentScript.parentElement; | |
| const data = [null,null]; | |
| Promise.all([ | |
| import("/docs/diffusers/pr_11450/en/_app/immutable/entry/start.4ce0a7ff.js"), | |
| import("/docs/diffusers/pr_11450/en/_app/immutable/entry/app.71328ef6.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 232], | |
| data, | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 91 kB
- Xet hash:
- e27b23d4685de12d634a28fe38c195f740733e952b11590b9be8706980661c74
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.