Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"ModularPipeline","local":"modularpipeline","sections":[{"title":"Create ModularPipelineBlocks","local":"create-modularpipelineblocks","sections":[],"depth":2},{"title":"Modular Repo","local":"modular-repo","sections":[],"depth":2},{"title":"Creating a ModularPipeline from ModularPipelineBlocks","local":"creating-a-modularpipeline-from-modularpipelineblocks","sections":[],"depth":2},{"title":"Creating a ModularPipeline with from_pretrained","local":"creating-a-modularpipeline-with-frompretrained","sections":[],"depth":2},{"title":"Loading components into a ModularPipeline","local":"loading-components-into-a-modularpipeline","sections":[],"depth":2},{"title":"Modifying Loading Specs","local":"modifying-loading-specs","sections":[],"depth":2},{"title":"Updating components in a ModularPipeline","local":"updating-components-in-a-modularpipeline","sections":[],"depth":2},{"title":"Customizing Guidance Techniques","local":"customizing-guidance-techniques","sections":[{"title":"Modify Parameters of the Same Guider Type","local":"modify-parameters-of-the-same-guider-type","sections":[],"depth":4},{"title":"Switch to a Different Guider Type","local":"switch-to-a-different-guider-type","sections":[],"depth":4},{"title":"Loading Custom Guiders from Hub","local":"loading-custom-guiders-from-hub","sections":[],"depth":4},{"title":"Upload Custom Guider to Hub for Easy Loading & Sharing","local":"upload-custom-guider-to-hub-for-easy-loading--sharing","sections":[],"depth":4}],"depth":2},{"title":"Running a ModularPipeline","local":"running-a-modularpipeline","sections":[{"title":"Text-to-Image, Image-to-Image, and Inpainting","local":"text-to-image-image-to-image-and-inpainting","sections":[],"depth":4},{"title":"ControlNet","local":"controlnet","sections":[],"depth":4},{"title":"IP-Adapter","local":"ip-adapter","sections":[],"depth":4}],"depth":2}],"depth":1}"> | |
| <link href="/docs/diffusers/pr_11797/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/entry/start.2e1ee9a7.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/chunks/scheduler.8c3d61f6.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/chunks/singletons.702cc21f.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/chunks/index.0997d446.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/chunks/paths.ec7f49a7.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/entry/app.87c95e21.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/chunks/index.da70eac4.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/nodes/0.7e207bc6.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/chunks/each.e59479a4.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/nodes/232.ec32ac09.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/chunks/Tip.1d9b8c37.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/chunks/CodeBlock.a9c4becf.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/chunks/getInferenceSnippets.725ed3d4.js"> | |
| <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/chunks/HfOption.6c3b4e77.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"ModularPipeline","local":"modularpipeline","sections":[{"title":"Create ModularPipelineBlocks","local":"create-modularpipelineblocks","sections":[],"depth":2},{"title":"Modular Repo","local":"modular-repo","sections":[],"depth":2},{"title":"Creating a ModularPipeline from ModularPipelineBlocks","local":"creating-a-modularpipeline-from-modularpipelineblocks","sections":[],"depth":2},{"title":"Creating a ModularPipeline with from_pretrained","local":"creating-a-modularpipeline-with-frompretrained","sections":[],"depth":2},{"title":"Loading components into a ModularPipeline","local":"loading-components-into-a-modularpipeline","sections":[],"depth":2},{"title":"Modifying Loading Specs","local":"modifying-loading-specs","sections":[],"depth":2},{"title":"Updating components in a ModularPipeline","local":"updating-components-in-a-modularpipeline","sections":[],"depth":2},{"title":"Customizing Guidance Techniques","local":"customizing-guidance-techniques","sections":[{"title":"Modify Parameters of the Same Guider Type","local":"modify-parameters-of-the-same-guider-type","sections":[],"depth":4},{"title":"Switch to a Different Guider Type","local":"switch-to-a-different-guider-type","sections":[],"depth":4},{"title":"Loading Custom Guiders from Hub","local":"loading-custom-guiders-from-hub","sections":[],"depth":4},{"title":"Upload Custom Guider to Hub for Easy Loading & Sharing","local":"upload-custom-guider-to-hub-for-easy-loading--sharing","sections":[],"depth":4}],"depth":2},{"title":"Running a ModularPipeline","local":"running-a-modularpipeline","sections":[{"title":"Text-to-Image, Image-to-Image, and Inpainting","local":"text-to-image-image-to-image-and-inpainting","sections":[],"depth":4},{"title":"ControlNet","local":"controlnet","sections":[],"depth":4},{"title":"IP-Adapter","local":"ip-adapter","sections":[],"depth":4}],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="modularpipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#modularpipeline"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>ModularPipeline</span></h1> <div class="course-tip course-tip-orange bg-gradient-to-br dark:bg-gradient-to-r before:border-orange-500 dark:before:border-orange-800 from-orange-50 dark:from-gray-900 to-white dark:to-gray-950 border border-orange-50 text-orange-700 dark:text-gray-400"><p data-svelte-h="svelte-11qh5l3">🧪 <strong>Experimental Feature</strong>: Modular Diffusers is an experimental feature we are actively developing. The API may be subject to breaking changes.</p></div> <p data-svelte-h="svelte-14h81hv"><code>ModularPipeline</code> is the main interface for end users to run pipelines in Modular Diffusers. It takes pipeline blocks and converts them into a runnable pipeline that can load models and execute the computation steps.</p> <p data-svelte-h="svelte-f6oz5n">In this guide, we will focus on how to build pipelines using the blocks we officially support at diffusers 🧨. We’ll cover how to use predefined blocks and convert them into a <code>ModularPipeline</code> for execution.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-7o9ugj">This guide shows you how to use predefined blocks. If you want to learn how to create your own pipeline blocks, see the <a href="pipeline_block.md">PipelineBlock guide</a> for creating individual blocks, and the multi-block guides for connecting them together:</p> <ul data-svelte-h="svelte-o8tp3o"><li><a href="sequential_pipeline_blocks.md">SequentialPipelineBlocks</a> (for linear workflows)</li> <li><a href="loop_sequential_pipeline_blocks.md">LoopSequentialPipelineBlocks</a> (for iterative workflows)</li> <li><a href="auto_pipeline_blocks.md">AutoPipelineBlocks</a> (for conditional workflows)</li></ul> <p data-svelte-h="svelte-1fb5414">For information on how data flows through pipelines, see the <a href="modular_diffusers_states.md">PipelineState and BlockState guide</a>.</p></div> <h2 class="relative group"><a id="create-modularpipelineblocks" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#create-modularpipelineblocks"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Create ModularPipelineBlocks</span></h2> <p data-svelte-h="svelte-1x8pk7u">In Modular Diffusers system, you build pipelines using Pipeline blocks. Pipeline Blocks are fundamental building blocks - they define what components, inputs/outputs, and computation logics are needed. They are designed to be assembled into workflows for tasks such as image generation, video creation, and inpainting. But they are just definitions and don’t actually run anything. To execute blocks, you need to put them into a <code>ModularPipeline</code>. We’ll first learn how to create predefined blocks here before talking about how to run them using <code>ModularPipeline</code>.</p> <p data-svelte-h="svelte-13lsz2q">All pipeline blocks inherit from the base class <code>ModularPipelineBlocks</code>, including:</p> <ul data-svelte-h="svelte-fv7zkr"><li><code>PipelineBlock</code>: The most granular block - you define the input/output/components requirements and computation logic.</li> <li><code>SequentialPipelineBlocks</code>: A multi-block composed of multiple blocks that run sequentially, passing outputs as inputs to the next block.</li> <li><code>LoopSequentialPipelineBlocks</code>: A special type of <code>SequentialPipelineBlocks</code> that runs the same sequence of blocks multiple times (loops), typically used for iterative processes like denoising steps in diffusion models.</li> <li><code>AutoPipelineBlocks</code>: A multi-block composed of multiple blocks that are selected at runtime based on the inputs.</li></ul> <p data-svelte-h="svelte-6rc13t">It is very easy to use a <code>ModularPipelineBlocks</code> officially supported in 🧨 Diffusers</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> diffusers.modular_pipelines.stable_diffusion_xl <span class="hljs-keyword">import</span> StableDiffusionXLTextEncoderStep | |
| text_encoder_block = StableDiffusionXLTextEncoderStep()<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-j9znyd">This is a single <code>PipelineBlock</code>. You’ll see that this text encoder block uses 2 text_encoders, 2 tokenizers as well as a guider component. It takes user inputs such as <code>prompt</code> and <code>negative_prompt</code>, and return text embeddings outputs such as <code>prompt_embeds</code> and <code>negative_prompt_embeds</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>text_encoder_block | |
| StableDiffusionXLTextEncoderStep( | |
| Class: PipelineBlock | |
| Description: Text Encoder step that generate text_embeddings to guide the image generation | |
| Components: | |
| text_encoder (`CLIPTextModel`) | |
| text_encoder_2 (`CLIPTextModelWithProjection`) | |
| tokenizer (`CLIPTokenizer`) | |
| tokenizer_2 (`CLIPTokenizer`) | |
| guider (`ClassifierFreeGuidance`) | |
| Configs: | |
| force_zeros_for_empty_prompt (default: <span class="hljs-literal">True</span>) | |
| Inputs: | |
| prompt=<span class="hljs-literal">None</span>, prompt_2=<span class="hljs-literal">None</span>, negative_prompt=<span class="hljs-literal">None</span>, negative_prompt_2=<span class="hljs-literal">None</span>, cross_attention_kwargs=<span class="hljs-literal">None</span>, clip_skip=<span class="hljs-literal">None</span> | |
| Intermediates: | |
| - outputs: prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds | |
| )<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-f8mebp">More commonly, you need multiple blocks to build your workflow. You can create a <code>SequentialPipelineBlocks</code> using block class presets from 🧨 Diffusers. <code>TEXT2IMAGE_BLOCKS</code> is a dict containing all the blocks needed for text-to-image generation.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> diffusers.modular_pipelines <span class="hljs-keyword">import</span> SequentialPipelineBlocks | |
| <span class="hljs-keyword">from</span> diffusers.modular_pipelines.stable_diffusion_xl <span class="hljs-keyword">import</span> TEXT2IMAGE_BLOCKS | |
| t2i_blocks = SequentialPipelineBlocks.from_blocks_dict(TEXT2IMAGE_BLOCKS)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-exn8w9">This creates a <code>SequentialPipelineBlocks</code>. Unlike the <code>text_encoder_block</code> we saw earlier, this is a multi-block and its <code>sub_blocks</code> attribute contains a list of other blocks (text_encoder, input, set_timesteps, prepare_latents, prepare_added_con, denoise, decode). Its requirements for components, inputs, and intermediate inputs are combined from these blocks that compose it. At runtime, it executes its sub-blocks sequentially and passes the pipeline state from one block to another.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>t2i_blocks | |
| SequentialPipelineBlocks( | |
| Class: ModularPipelineBlocks | |
| Description: | |
| Components: | |
| text_encoder (`CLIPTextModel`) | |
| text_encoder_2 (`CLIPTextModelWithProjection`) | |
| tokenizer (`CLIPTokenizer`) | |
| tokenizer_2 (`CLIPTokenizer`) | |
| guider (`ClassifierFreeGuidance`) | |
| scheduler (`EulerDiscreteScheduler`) | |
| unet (`UNet2DConditionModel`) | |
| vae (`AutoencoderKL`) | |
| image_processor (`VaeImageProcessor`) | |
| Configs: | |
| force_zeros_for_empty_prompt (default: <span class="hljs-literal">True</span>) | |
| Sub-Blocks: | |
| [<span class="hljs-number">0</span>] text_encoder (StableDiffusionXLTextEncoderStep) | |
| Description: Text Encoder step that generate text_embeddings to guide the image generation | |
| [<span class="hljs-number">1</span>] <span class="hljs-built_in">input</span> (StableDiffusionXLInputStep) | |
| Description: Input processing step that: | |
| <span class="hljs-number">1.</span> Determines `batch_size` <span class="hljs-keyword">and</span> `dtype` based on `prompt_embeds` | |
| <span class="hljs-number">2.</span> Adjusts <span class="hljs-built_in">input</span> tensor shapes based on `batch_size` (number of prompts) <span class="hljs-keyword">and</span> `num_images_per_prompt` | |
| All <span class="hljs-built_in">input</span> tensors are expected to have either batch_size=<span class="hljs-number">1</span> <span class="hljs-keyword">or</span> <span class="hljs-keyword">match</span> the batch_size | |
| of prompt_embeds. The tensors will be duplicated across the batch dimension to | |
| have a final batch_size of batch_size * num_images_per_prompt. | |
| [<span class="hljs-number">2</span>] set_timesteps (StableDiffusionXLSetTimestepsStep) | |
| Description: Step that sets the schedule<span class="hljs-string">r's timesteps for inference | |
| [3] prepare_latents (StableDiffusionXLPrepareLatentsStep) | |
| Description: Prepare latents step that prepares the latents for the text-to-image generation process | |
| [4] prepare_add_cond (StableDiffusionXLPrepareAdditionalConditioningStep) | |
| Description: Step that prepares the additional conditioning for the text-to-image generation process | |
| [5] denoise (StableDiffusionXLDenoiseStep) | |
| Description: Denoise step that iteratively denoise the latents. | |
| Its loop logic is defined in `StableDiffusionXLDenoiseLoopWrapper.__call__` method | |
| At each iteration, it runs blocks defined in `sub_blocks` sequencially: | |
| - `StableDiffusionXLLoopBeforeDenoiser` | |
| - `StableDiffusionXLLoopDenoiser` | |
| - `StableDiffusionXLLoopAfterDenoiser` | |
| This block supports both text2img and img2img tasks. | |
| [6] decode (StableDiffusionXLDecodeStep) | |
| Description: Step that decodes the denoised latents into images | |
| )</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-143wuw6">This is the block classes preset (<code>TEXT2IMAGE_BLOCKS</code>) we used: It is just a dictionary that maps names to ModularPipelineBlocks classes</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>TEXT2IMAGE_BLOCKS | |
| InsertableDict([ | |
| <span class="hljs-number">0</span>: (<span class="hljs-string">'text_encoder'</span>, <<span class="hljs-keyword">class</span> <span class="hljs-string">'diffusers.modular_pipelines.stable_diffusion_xl.encoders.StableDiffusionXLTextEncoderStep'</span>>), | |
| <span class="hljs-number">1</span>: (<span class="hljs-string">'input'</span>, <<span class="hljs-keyword">class</span> <span class="hljs-string">'diffusers.modular_pipelines.stable_diffusion_xl.before_denoise.StableDiffusionXLInputStep'</span>>), | |
| <span class="hljs-number">2</span>: (<span class="hljs-string">'set_timesteps'</span>, <<span class="hljs-keyword">class</span> <span class="hljs-string">'diffusers.modular_pipelines.stable_diffusion_xl.before_denoise.StableDiffusionXLSetTimestepsStep'</span>>), | |
| <span class="hljs-number">3</span>: (<span class="hljs-string">'prepare_latents'</span>, <<span class="hljs-keyword">class</span> <span class="hljs-string">'diffusers.modular_pipelines.stable_diffusion_xl.before_denoise.StableDiffusionXLPrepareLatentsStep'</span>>), | |
| <span class="hljs-number">4</span>: (<span class="hljs-string">'prepare_add_cond'</span>, <<span class="hljs-keyword">class</span> <span class="hljs-string">'diffusers.modular_pipelines.stable_diffusion_xl.before_denoise.StableDiffusionXLPrepareAdditionalConditioningStep'</span>>), | |
| <span class="hljs-number">5</span>: (<span class="hljs-string">'denoise'</span>, <<span class="hljs-keyword">class</span> <span class="hljs-string">'diffusers.modular_pipelines.stable_diffusion_xl.denoise.StableDiffusionXLDenoiseLoop'</span>>), | |
| <span class="hljs-number">6</span>: (<span class="hljs-string">'decode'</span>, <<span class="hljs-keyword">class</span> <span class="hljs-string">'diffusers.modular_pipelines.stable_diffusion_xl.decoders.StableDiffusionXLDecodeStep'</span>>) | |
| ])<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-lp03vj">When we create a <code>SequentialPipelineBlocks</code> from this preset, it instantiates each block class into actual block objects. Its <code>sub_blocks</code> attribute now contains these instantiated objects:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>t2i_blocks.sub_blocks | |
| InsertableDict([ | |
| <span class="hljs-number">0</span>: (<span class="hljs-string">'text_encoder'</span>, <obj <span class="hljs-string">'diffusers.modular_pipelines.stable_diffusion_xl.encoders.StableDiffusionXLTextEncoderStep'</span>>), | |
| <span class="hljs-number">1</span>: (<span class="hljs-string">'input'</span>, <obj <span class="hljs-string">'diffusers.modular_pipelines.stable_diffusion_xl.before_denoise.StableDiffusionXLInputStep'</span>>), | |
| <span class="hljs-number">2</span>: (<span class="hljs-string">'set_timesteps'</span>, <obj <span class="hljs-string">'diffusers.modular_pipelines.stable_diffusion_xl.before_denoise.StableDiffusionXLSetTimestepsStep'</span>>), | |
| <span class="hljs-number">3</span>: (<span class="hljs-string">'prepare_latents'</span>, <obj <span class="hljs-string">'diffusers.modular_pipelines.stable_diffusion_xl.before_denoise.StableDiffusionXLPrepareLatentsStep'</span>>), | |
| <span class="hljs-number">4</span>: (<span class="hljs-string">'prepare_add_cond'</span>, <obj <span class="hljs-string">'diffusers.modular_pipelines.stable_diffusion_xl.before_denoise.StableDiffusionXLPrepareAdditionalConditioningStep'</span>>), | |
| <span class="hljs-number">5</span>: (<span class="hljs-string">'denoise'</span>, <obj <span class="hljs-string">'diffusers.modular_pipelines.stable_diffusion_xl.denoise.StableDiffusionXLDenoiseStep'</span>>), | |
| <span class="hljs-number">6</span>: (<span class="hljs-string">'decode'</span>, <obj <span class="hljs-string">'diffusers.modular_pipelines.stable_diffusion_xl.decoders.StableDiffusionXLDecodeStep'</span>>) | |
| ])<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1j1t6iv">Note that both the block classes preset and the <code>sub_blocks</code> attribute are <code>InsertableDict</code> objects. This is a custom dictionary that extends <code>OrderedDict</code> with the ability to insert items at specific positions. You can perform all standard dictionary operations (get, set, delete) plus insert items at any index, which is particularly useful for reordering or inserting blocks in the middle of a pipeline.</p> <p data-svelte-h="svelte-14fxc5m"><strong>Add a block:</strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># BLOCKS is dict of block classes, you need to add class to it</span> | |
| BLOCKS.insert(<span class="hljs-string">"block_name"</span>, BlockClass, index) | |
| <span class="hljs-comment"># sub_blocks attribute contains instance, add a block instance to the attribute</span> | |
| t2i_blocks.sub_blocks.insert(<span class="hljs-string">"block_name"</span>, block_instance, index)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-11cgoy5"><strong>Remove a block:</strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># remove a block class from preset</span> | |
| BLOCKS.pop(<span class="hljs-string">"text_encoder"</span>) | |
| <span class="hljs-comment"># split out a block instance on its own</span> | |
| text_encoder_block = t2i_blocks.sub_blocks.pop(<span class="hljs-string">"text_encoder"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-weuwnx"><strong>Swap block:</strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># Replace block class in preset</span> | |
| BLOCKS[<span class="hljs-string">"prepare_latents"</span>] = CustomPrepareLatents | |
| <span class="hljs-comment"># Replace in sub_blocks attribute using an block instance</span> | |
| t2i_blocks.sub_blocks[<span class="hljs-string">"prepare_latents"</span>] = CustomPrepareLatents()<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-nsxsu4">This means you can mix-and-match blocks in very flexible ways. Let’s see some real examples:</p> <p data-svelte-h="svelte-duhu0z"><strong>Example 1: Adding IP-Adapter to the Block Classes Preset</strong> | |
| Let’s make a new block classes preset by insert IP-Adapter at index 0 (before the text_encoder block), and create a text-to-image pipeline with IP-Adapter support:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> diffusers.modular_pipelines.stable_diffusion_xl <span class="hljs-keyword">import</span> StableDiffusionXLAutoIPAdapterStep | |
| CUSTOM_BLOCKS = TEXT2IMAGE_BLOCKS.copy() | |
| <span class="hljs-comment"># CUSTOM_BLOCKS is now a preset including ip_adapter</span> | |
| CUSTOM_BLOCKS.insert(<span class="hljs-string">"ip_adapter"</span>, StableDiffusionXLAutoIPAdapterStep, <span class="hljs-number">0</span>) | |
| <span class="hljs-comment"># create a blocks isntance from the preset</span> | |
| custom_blocks = SequentialPipelineBlocks.from_blocks_dict(CUSTOM_BLOCKS)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-14jelb5"><strong>Example 2: Extracting a block from a multi-block</strong> | |
| You can extract a block instance from the multi-block to use it independently. A common pattern is to use text_encoder to process prompts once, then reuse the text embeddings outputs to generate multiple images with different settings (schedulers, seeds, inference steps). We can do this by simply extracting the text_encoder block from the pipeline.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># this gives you StableDiffusionXLTextEncoderStep()</span> | |
| <span class="hljs-meta">>>> </span>text_encoder_blocks = t2i_blocks.sub_blocks.pop(<span class="hljs-string">"text_encoder"</span>) | |
| <span class="hljs-meta">>>> </span>text_encoder_blocks<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-u4kp4u">The multi-block now has fewer components and no longer has the <code>text_encoder</code> block. If you check its docstring <code>t2i_blocks.doc</code>, you will see that it no longer accepts <code>prompt</code> as input - you will need to pass the embeddings instead.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>t2i_blocks | |
| SequentialPipelineBlocks( | |
| Class: ModularPipelineBlocks | |
| Description: | |
| Components: | |
| scheduler (`EulerDiscreteScheduler`) | |
| guider (`ClassifierFreeGuidance`) | |
| unet (`UNet2DConditionModel`) | |
| vae (`AutoencoderKL`) | |
| image_processor (`VaeImageProcessor`) | |
| Blocks: | |
| [<span class="hljs-number">0</span>] <span class="hljs-built_in">input</span> (StableDiffusionXLInputStep) | |
| Description: Input processing step that: | |
| <span class="hljs-number">1.</span> Determines `batch_size` <span class="hljs-keyword">and</span> `dtype` based on `prompt_embeds` | |
| <span class="hljs-number">2.</span> Adjusts <span class="hljs-built_in">input</span> tensor shapes based on `batch_size` (number of prompts) <span class="hljs-keyword">and</span> `num_images_per_prompt` | |
| All <span class="hljs-built_in">input</span> tensors are expected to have either batch_size=<span class="hljs-number">1</span> <span class="hljs-keyword">or</span> <span class="hljs-keyword">match</span> the batch_size | |
| of prompt_embeds. The tensors will be duplicated across the batch dimension to | |
| have a final batch_size of batch_size * num_images_per_prompt. | |
| [<span class="hljs-number">1</span>] set_timesteps (StableDiffusionXLSetTimestepsStep) | |
| Description: Step that sets the schedule<span class="hljs-string">r's timesteps for inference | |
| [2] prepare_latents (StableDiffusionXLPrepareLatentsStep) | |
| Description: Prepare latents step that prepares the latents for the text-to-image generation process | |
| [3] prepare_add_cond (StableDiffusionXLPrepareAdditionalConditioningStep) | |
| Description: Step that prepares the additional conditioning for the text-to-image generation process | |
| [4] denoise (StableDiffusionXLDenoiseLoop) | |
| Description: Denoise step that iteratively denoise the latents. | |
| Its loop logic is defined in `StableDiffusionXLDenoiseLoopWrapper.__call__` method | |
| At each iteration, it runs blocks defined in `blocks` sequencially: | |
| - `StableDiffusionXLLoopBeforeDenoiser` | |
| - `StableDiffusionXLLoopDenoiser` | |
| - `StableDiffusionXLLoopAfterDenoiser` | |
| [5] decode (StableDiffusionXLDecodeStep) | |
| Description: Step that decodes the denoised latents into images | |
| )</span><!-- HTML_TAG_END --></pre></div> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-1iu2bgl">💡 You can find all the block classes presets we support for each model in <code>ALL_BLOCKS</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># For Stable Diffusion XL</span> | |
| <span class="hljs-keyword">from</span> diffusers.modular_pipelines.stable_diffusion_xl <span class="hljs-keyword">import</span> ALL_BLOCKS | |
| ALL_BLOCKS | |
| <span class="hljs-comment"># For other models...</span> | |
| <span class="hljs-keyword">from</span> diffusers.modular_pipelines.<model_name> <span class="hljs-keyword">import</span> ALL_BLOCKS<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-8bbxo4">Each model provides a dictionary that maps all supported tasks/techniques to their corresponding block classes presets. For SDXL, it is</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->ALL_BLOCKS = { | |
| <span class="hljs-string">"text2img"</span>: TEXT2IMAGE_BLOCKS, | |
| <span class="hljs-string">"img2img"</span>: IMAGE2IMAGE_BLOCKS, | |
| <span class="hljs-string">"inpaint"</span>: INPAINT_BLOCKS, | |
| <span class="hljs-string">"controlnet"</span>: CONTROLNET_BLOCKS, | |
| <span class="hljs-string">"ip_adapter"</span>: IP_ADAPTER_BLOCKS, | |
| <span class="hljs-string">"auto"</span>: AUTO_BLOCKS, | |
| }<!-- HTML_TAG_END --></pre></div></div> <p data-svelte-h="svelte-1ekh1wv">This covers the essentials of pipeline blocks! Like we have already mentioned, <strong>pipeline blocks are not runnable by themselves</strong>. They are essentially <strong>“definitions”</strong> - they define the specifications and computational steps for a pipeline, but they do not contain any model states. To actually run them, you need to convert them into a <code>ModularPipeline</code> object.</p> <h2 class="relative group"><a id="modular-repo" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#modular-repo"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Modular Repo</span></h2> <p data-svelte-h="svelte-1kao3k1">To convert blocks into a runnable pipeline, you may need a repository if your blocks contain <strong>pretrained components</strong> (models with checkpoints that need to be loaded from the Hub). Pipeline blocks define what components they need (like a UNet, text encoder, etc.), as well as how to create them: components can be either created using <strong>from_pretrained</strong> method (with checkpoints) or <strong>from_config</strong> (initialized from scratch with default configuration, usually stateless like a guider or scheduler).</p> <p data-svelte-h="svelte-tegpo4">If your pipeline contains <strong>pretrained components</strong>, you typically need to use a repository to provide the loading specifications and metadata.</p> <p data-svelte-h="svelte-17zvy7e"><code>ModularPipeline</code> works specifically with modular repositories, which offer more flexibility in component loading compared to traditional repositories. You can find an example modular repo <a href="https://huggingface.co/YiYiXu/modular-diffdiff" rel="nofollow">here</a>.</p> <p data-svelte-h="svelte-1ycj0wy">A <code>DiffusionPipeline</code> defines <code>model_index.json</code> to configure its components. However, repositories for Modular Diffusers work with <code>modular_model_index.json</code>. Let’s walk through the differences here.</p> <p data-svelte-h="svelte-1vmw4gr">In standard <code>model_index.json</code>, each component entry is a <code>(library, class)</code> tuple:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-string">"text_encoder"</span>: [ | |
| <span class="hljs-string">"transformers"</span>, | |
| <span class="hljs-string">"CLIPTextModel"</span> | |
| ],<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-gxancp">In <code>modular_model_index.json</code>, each component entry contains 3 elements: <code>(library, class, loading_specs_dict)</code></p> <ul data-svelte-h="svelte-107f998"><li><code>library</code> and <code>class</code>: Information about the actual component loaded in the pipeline at the time of saving (will be <code>null</code> if not loaded)</li> <li><code>loading_specs_dict</code>: A dictionary containing all information required to load this component, including <code>repo</code>, <code>revision</code>, <code>subfolder</code>, <code>variant</code>, and <code>type_hint</code>.</li></ul> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-string">"text_encoder"</span>: [ | |
| null, <span class="hljs-comment"># library of actual loaded component (same as in model_index.json)</span> | |
| null, <span class="hljs-comment"># class of actual loaded componenet (same as in model_index.json)</span> | |
| { <span class="hljs-comment"># loading specs map (unique to modular_model_index.json)</span> | |
| <span class="hljs-string">"repo"</span>: <span class="hljs-string">"stabilityai/stable-diffusion-xl-base-1.0"</span>, <span class="hljs-comment"># can be a different repo</span> | |
| <span class="hljs-string">"revision"</span>: null, | |
| <span class="hljs-string">"subfolder"</span>: <span class="hljs-string">"text_encoder"</span>, | |
| <span class="hljs-string">"type_hint"</span>: [ <span class="hljs-comment"># (library, class) for the expected component</span> | |
| <span class="hljs-string">"transformers"</span>, | |
| <span class="hljs-string">"CLIPTextModel"</span> | |
| ], | |
| <span class="hljs-string">"variant"</span>: null | |
| } | |
| ],<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-7ittim">Unlike standard repositories where components must be in subfolders within the same repo, modular repositories can fetch components from different repositories based on the <code>loading_specs_dict</code>. e.g. the <code>text_encoder</code> component will be fetched from the “text_encoder” folder in <code>stabilityai/stable-diffusion-xl-base-1.0</code> while other components come from different repositories.</p> <h2 class="relative group"><a id="creating-a-modularpipeline-from-modularpipelineblocks" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#creating-a-modularpipeline-from-modularpipelineblocks"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Creating a ModularPipeline from ModularPipelineBlocks</span></h2> <p data-svelte-h="svelte-jtwth8">Each <code>ModularPipelineBlocks</code> has an <code>init_pipeline</code> method that can initialize a <code>ModularPipeline</code> object based on its component and configuration specifications.</p> <p data-svelte-h="svelte-1afy7a4">Let’s convert our <code>t2i_blocks</code> (which we created earlier) into a runnable <code>ModularPipeline</code>. We’ll use a <code>ComponentsManager</code> to handle device placement, memory management, and component reuse automatically:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># We already have this from earlier</span> | |
| t2i_blocks = SequentialPipelineBlocks.from_blocks_dict(TEXT2IMAGE_BLOCKS) | |
| <span class="hljs-comment"># Now convert it to a ModularPipeline</span> | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> ComponentsManager | |
| modular_repo_id = <span class="hljs-string">"YiYiXu/modular-loader-t2i-0704"</span> | |
| components = ComponentsManager() | |
| t2i_pipeline = t2i_blocks.init_pipeline(modular_repo_id, components_manager=components)<!-- HTML_TAG_END --></pre></div> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-w13he7">💡 <strong>ComponentsManager</strong> is the model registry and management system in diffusers, it track all the models in one place and let you add, remove and reuse them across different workflows in most efficient way. Without it, you’d need to manually manage GPU memory, device placement, and component sharing between workflows. See the <a href="components_manager.md">Components Manager guide</a> for detailed information.</p></div> <p data-svelte-h="svelte-1dvdh2z">The <code>init_pipeline()</code> method creates a ModularPipeline and loads component specifications from the repository’s <code>modular_model_index.json</code> file, but doesn’t load the actual models yet.</p> <h2 class="relative group"><a id="creating-a-modularpipeline-with-frompretrained" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#creating-a-modularpipeline-with-frompretrained"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Creating a ModularPipeline with from_pretrained</span></h2> <p data-svelte-h="svelte-1711rzq">You can create a <code>ModularPipeline</code> from a HuggingFace Hub repository with <code>from_pretrained</code> method, as long as it’s a modular repo:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> ModularPipeline, ComponentsManager | |
| components = ComponentsManager() | |
| pipeline = ModularPipeline.from_pretrained(<span class="hljs-string">"YiYiXu/modular-loader-t2i-0704"</span>, components_manager=components)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-gtmcub">Loading custom code is also supported:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> ModularPipeline, ComponentsManager | |
| components = ComponentsManager() | |
| modular_repo_id = <span class="hljs-string">"YiYiXu/modular-diffdiff-0704"</span> | |
| diffdiff_pipeline = ModularPipeline.from_pretrained(modular_repo_id, trust_remote_code=<span class="hljs-literal">True</span>, components_manager=components)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-aad207">This modular repository contains custom code. The folder contains these files:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->modular-<span class="hljs-keyword">diffdiff-0704/ | |
| </span>├── <span class="hljs-keyword">block.py </span> <span class="hljs-comment"># Custom pipeline blocks implementation</span> | |
| ├── <span class="hljs-built_in">config</span>.<span class="hljs-keyword">json </span> <span class="hljs-comment"># Pipeline configuration and auto_map</span> | |
| └── modular_model_index.<span class="hljs-keyword">json </span> <span class="hljs-comment"># Component loading specifications</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1juzgj">The <a href="https://huggingface.co/YiYiXu/modular-diffdiff-0704/blob/main/config.json" rel="nofollow"><code>config.json</code></a> file defines a custom <code>DiffDiffBlocks</code> class and points to its implementation:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-punctuation">{</span> | |
| <span class="hljs-attr">"_class_name"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"DiffDiffBlocks"</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"auto_map"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span> | |
| <span class="hljs-attr">"ModularPipelineBlocks"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"block.DiffDiffBlocks"</span> | |
| <span class="hljs-punctuation">}</span> | |
| <span class="hljs-punctuation">}</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1vn1o8k">The <code>auto_map</code> tells the pipeline where to find the custom blocks definition - in this case, it’s looking for <code>DiffDiffBlocks</code> in the <code>block.py</code> file. The actual <code>DiffDiffBlocks</code> class is defined in <a href="https://huggingface.co/YiYiXu/modular-diffdiff-0704/blob/main/block.py" rel="nofollow"><code>block.py</code></a> within the repository.</p> <p data-svelte-h="svelte-1ywa8pm">When <code>diffdiff_pipeline.blocks</code> is created, it’s based on the <code>DiffDiffBlocks</code> definition from the custom code in the repository, allowing you to use specialized blocks that aren’t part of the standard diffusers library.</p> <h2 class="relative group"><a id="loading-components-into-a-modularpipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#loading-components-into-a-modularpipeline"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Loading components into a ModularPipeline</span></h2> <p data-svelte-h="svelte-4mvx0e">Unlike <code>DiffusionPipeline</code>, when you create a <code>ModularPipeline</code> instance (whether using <code>from_pretrained</code> or converting from pipeline blocks), its components aren’t loaded automatically. You need to explicitly load model components using <code>load_default_components</code> or <code>load_components(names=..,)</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># This will load ALL the expected components into pipeline</span> | |
| <span class="hljs-keyword">import</span> torch | |
| t2i_pipeline.load_default_components(torch_dtype=torch.float16) | |
| t2i_pipeline.to(<span class="hljs-string">"cuda"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1c9iz5d">All expected components are now loaded into the pipeline. You can also partially load specific components using the <code>names</code> argument. For example, to only load unet and vae:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>t2i_pipeline.load_components(names=[<span class="hljs-string">"unet"</span>, <span class="hljs-string">"vae"</span>], torch_dtype=torch.float16)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-18uy5ju">You can inspect the pipeline’s loading status by simply printing the pipeline itself. It helps you understand what components are expected to load, which ones are already loaded, how they were loaded, and what loading specs are available. Let’s print out the <code>t2i_pipeline</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>t2i_pipeline | |
| StableDiffusionXLModularPipeline { | |
| <span class="hljs-string">"_blocks_class_name"</span>: <span class="hljs-string">"SequentialPipelineBlocks"</span>, | |
| <span class="hljs-string">"_class_name"</span>: <span class="hljs-string">"StableDiffusionXLModularPipeline"</span>, | |
| <span class="hljs-string">"_diffusers_version"</span>: <span class="hljs-string">"0.35.0.dev0"</span>, | |
| <span class="hljs-string">"force_zeros_for_empty_prompt"</span>: true, | |
| <span class="hljs-string">"scheduler"</span>: [ | |
| null, | |
| null, | |
| { | |
| <span class="hljs-string">"repo"</span>: <span class="hljs-string">"stabilityai/stable-diffusion-xl-base-1.0"</span>, | |
| <span class="hljs-string">"revision"</span>: null, | |
| <span class="hljs-string">"subfolder"</span>: <span class="hljs-string">"scheduler"</span>, | |
| <span class="hljs-string">"type_hint"</span>: [ | |
| <span class="hljs-string">"diffusers"</span>, | |
| <span class="hljs-string">"EulerDiscreteScheduler"</span> | |
| ], | |
| <span class="hljs-string">"variant"</span>: null | |
| } | |
| ], | |
| <span class="hljs-string">"text_encoder"</span>: [ | |
| null, | |
| null, | |
| { | |
| <span class="hljs-string">"repo"</span>: <span class="hljs-string">"stabilityai/stable-diffusion-xl-base-1.0"</span>, | |
| <span class="hljs-string">"revision"</span>: null, | |
| <span class="hljs-string">"subfolder"</span>: <span class="hljs-string">"text_encoder"</span>, | |
| <span class="hljs-string">"type_hint"</span>: [ | |
| <span class="hljs-string">"transformers"</span>, | |
| <span class="hljs-string">"CLIPTextModel"</span> | |
| ], | |
| <span class="hljs-string">"variant"</span>: null | |
| } | |
| ], | |
| <span class="hljs-string">"text_encoder_2"</span>: [ | |
| null, | |
| null, | |
| { | |
| <span class="hljs-string">"repo"</span>: <span class="hljs-string">"stabilityai/stable-diffusion-xl-base-1.0"</span>, | |
| <span class="hljs-string">"revision"</span>: null, | |
| <span class="hljs-string">"subfolder"</span>: <span class="hljs-string">"text_encoder_2"</span>, | |
| <span class="hljs-string">"type_hint"</span>: [ | |
| <span class="hljs-string">"transformers"</span>, | |
| <span class="hljs-string">"CLIPTextModelWithProjection"</span> | |
| ], | |
| <span class="hljs-string">"variant"</span>: null | |
| } | |
| ], | |
| <span class="hljs-string">"tokenizer"</span>: [ | |
| null, | |
| null, | |
| { | |
| <span class="hljs-string">"repo"</span>: <span class="hljs-string">"stabilityai/stable-diffusion-xl-base-1.0"</span>, | |
| <span class="hljs-string">"revision"</span>: null, | |
| <span class="hljs-string">"subfolder"</span>: <span class="hljs-string">"tokenizer"</span>, | |
| <span class="hljs-string">"type_hint"</span>: [ | |
| <span class="hljs-string">"transformers"</span>, | |
| <span class="hljs-string">"CLIPTokenizer"</span> | |
| ], | |
| <span class="hljs-string">"variant"</span>: null | |
| } | |
| ], | |
| <span class="hljs-string">"tokenizer_2"</span>: [ | |
| null, | |
| null, | |
| { | |
| <span class="hljs-string">"repo"</span>: <span class="hljs-string">"stabilityai/stable-diffusion-xl-base-1.0"</span>, | |
| <span class="hljs-string">"revision"</span>: null, | |
| <span class="hljs-string">"subfolder"</span>: <span class="hljs-string">"tokenizer_2"</span>, | |
| <span class="hljs-string">"type_hint"</span>: [ | |
| <span class="hljs-string">"transformers"</span>, | |
| <span class="hljs-string">"CLIPTokenizer"</span> | |
| ], | |
| <span class="hljs-string">"variant"</span>: null | |
| } | |
| ], | |
| <span class="hljs-string">"unet"</span>: [ | |
| <span class="hljs-string">"diffusers"</span>, | |
| <span class="hljs-string">"UNet2DConditionModel"</span>, | |
| { | |
| <span class="hljs-string">"repo"</span>: <span class="hljs-string">"RunDiffusion/Juggernaut-XL-v9"</span>, | |
| <span class="hljs-string">"revision"</span>: null, | |
| <span class="hljs-string">"subfolder"</span>: <span class="hljs-string">"unet"</span>, | |
| <span class="hljs-string">"type_hint"</span>: [ | |
| <span class="hljs-string">"diffusers"</span>, | |
| <span class="hljs-string">"UNet2DConditionModel"</span> | |
| ], | |
| <span class="hljs-string">"variant"</span>: <span class="hljs-string">"fp16"</span> | |
| } | |
| ], | |
| <span class="hljs-string">"vae"</span>: [ | |
| <span class="hljs-string">"diffusers"</span>, | |
| <span class="hljs-string">"AutoencoderKL"</span>, | |
| { | |
| <span class="hljs-string">"repo"</span>: <span class="hljs-string">"madebyollin/sdxl-vae-fp16-fix"</span>, | |
| <span class="hljs-string">"revision"</span>: null, | |
| <span class="hljs-string">"subfolder"</span>: null, | |
| <span class="hljs-string">"type_hint"</span>: [ | |
| <span class="hljs-string">"diffusers"</span>, | |
| <span class="hljs-string">"AutoencoderKL"</span> | |
| ], | |
| <span class="hljs-string">"variant"</span>: null | |
| } | |
| ] | |
| }<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-aj04nf">You can see all the <strong>pretrained components</strong> that will be loaded using <code>from_pretrained</code> method are listed as entries. Each entry contains 3 elements: <code>(library, class, loading_specs_dict)</code>:</p> <ul data-svelte-h="svelte-17lbc83"><li><strong><code>library</code> and <code>class</code></strong>: Show the actual loaded component info. If <code>null</code>, the component is not loaded yet.</li> <li><strong><code>loading_specs_dict</code></strong>: Contains all the information needed to load the component (repo, subfolder, variant, etc.)</li></ul> <p data-svelte-h="svelte-15bi1il">In this example:</p> <ul data-svelte-h="svelte-zwkdv4"><li><strong>Loaded components</strong>: <code>vae</code> and <code>unet</code> (their <code>library</code> and <code>class</code> fields show the actual loaded models)</li> <li><strong>Not loaded yet</strong>: <code>scheduler</code>, <code>text_encoder</code>, <code>text_encoder_2</code>, <code>tokenizer</code>, <code>tokenizer_2</code> (their <code>library</code> and <code>class</code> fields are <code>null</code>, but you can see their loading specs to know where they’ll be loaded from when you call <code>load_components()</code>)</li></ul> <p data-svelte-h="svelte-1o98bnb">You’re looking at essentailly the pipeline’s config dict that’s synced with the <code>modular_model_index.json</code> from the repository you used during <code>init_pipeline()</code> - it takes the loading specs that match the pipeline’s component requirements.</p> <p data-svelte-h="svelte-e0gfuv">For example, if your pipeline needs a <code>text_encoder</code> component, it will include the loading spec for <code>text_encoder</code> from the modular repo during the <code>init_pipeline</code>. If the pipeline doesn’t need a component (like <code>controlnet</code> in a basic text-to-image pipeline), that component won’t be included even if it exists in the modular repo.</p> <p data-svelte-h="svelte-q1s7rj">There are also a few properties that can provide a quick summary of component loading status:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># All components expected by the pipeline</span> | |
| <span class="hljs-meta">>>> </span>t2i_pipeline.component_names | |
| [<span class="hljs-string">'text_encoder'</span>, <span class="hljs-string">'text_encoder_2'</span>, <span class="hljs-string">'tokenizer'</span>, <span class="hljs-string">'tokenizer_2'</span>, <span class="hljs-string">'guider'</span>, <span class="hljs-string">'scheduler'</span>, <span class="hljs-string">'unet'</span>, <span class="hljs-string">'vae'</span>, <span class="hljs-string">'image_processor'</span>] | |
| <span class="hljs-comment"># Components that are not loaded yet (will be loaded with from_pretrained)</span> | |
| <span class="hljs-meta">>>> </span>t2i_pipeline.null_component_names | |
| [<span class="hljs-string">'text_encoder'</span>, <span class="hljs-string">'text_encoder_2'</span>, <span class="hljs-string">'tokenizer'</span>, <span class="hljs-string">'tokenizer_2'</span>, <span class="hljs-string">'scheduler'</span>] | |
| <span class="hljs-comment"># Components that will be loaded from pretrained models</span> | |
| <span class="hljs-meta">>>> </span>t2i_pipeline.pretrained_component_names | |
| [<span class="hljs-string">'text_encoder'</span>, <span class="hljs-string">'text_encoder_2'</span>, <span class="hljs-string">'tokenizer'</span>, <span class="hljs-string">'tokenizer_2'</span>, <span class="hljs-string">'scheduler'</span>, <span class="hljs-string">'unet'</span>, <span class="hljs-string">'vae'</span>] | |
| <span class="hljs-comment"># Components that are created with default config (no repo needed)</span> | |
| <span class="hljs-meta">>>> </span>t2i_pipeline.config_component_names | |
| [<span class="hljs-string">'guider'</span>, <span class="hljs-string">'image_processor'</span>]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-u41c9d">From config components (like <code>guider</code> and <code>image_processor</code>) are not included in the pipeline output above because they don’t need loading specs - they’re already initialized during pipeline creation. You can see this because they’re not listed in <code>null_component_names</code>.</p> <h2 class="relative group"><a id="modifying-loading-specs" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#modifying-loading-specs"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Modifying Loading Specs</span></h2> <p data-svelte-h="svelte-14jnawd">When you call <code>pipeline.load_components(names=)</code> or <code>pipeline.load_default_components()</code>, it uses the loading specs from the modular repository’s <code>modular_model_index.json</code>. You can change where components are loaded from by modifying the <code>modular_model_index.json</code> in the repository. Just find the file on the Hub and click edit - you can change any field in the loading specs: <code>repo</code>, <code>subfolder</code>, <code>variant</code>, <code>revision</code>, etc.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># Original spec in modular_model_index.json</span> | |
| <span class="hljs-string">"unet"</span>: [ | |
| null, null, | |
| { | |
| <span class="hljs-string">"repo"</span>: <span class="hljs-string">"stabilityai/stable-diffusion-xl-base-1.0"</span>, | |
| <span class="hljs-string">"subfolder"</span>: <span class="hljs-string">"unet"</span>, | |
| <span class="hljs-string">"variant"</span>: <span class="hljs-string">"fp16"</span> | |
| } | |
| ] | |
| <span class="hljs-comment"># Modified spec - changed repo, subfolder, and variant</span> | |
| <span class="hljs-string">"unet"</span>: [ | |
| null, null, | |
| { | |
| <span class="hljs-string">"repo"</span>: <span class="hljs-string">"RunDiffusion/Juggernaut-XL-v9"</span>, | |
| <span class="hljs-string">"subfolder"</span>: <span class="hljs-string">"unet"</span>, | |
| <span class="hljs-string">"variant"</span>: <span class="hljs-string">"fp16"</span> | |
| } | |
| ]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-ewnlrw">Now if you create a pipeline using the same blocks and updated repository, it will by default load from the new repository.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->pipeline = ModularPipeline.from_pretrained(<span class="hljs-string">"YiYiXu/modular-loader-t2i-0704"</span>, components_manager=components) | |
| pipeline.load_components(names=<span class="hljs-string">"unet"</span>)<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="updating-components-in-a-modularpipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#updating-components-in-a-modularpipeline"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Updating components in a ModularPipeline</span></h2> <p data-svelte-h="svelte-142s7uo">Similar to <code>DiffusionPipeline</code>, you can load components separately to replace the default ones in the pipeline. In Modular Diffusers, the approach depends on the component type:</p> <ul data-svelte-h="svelte-yp3jh"><li><strong>Pretrained components</strong> (<code>default_creation_method='from_pretrained'</code>): Must use <code>ComponentSpec</code> to load them to update the existing one.</li> <li><strong>Config components</strong> (<code>default_creation_method='from_config'</code>): These are components that don’t need loading specs - they’re created during pipeline initialization with default config. To update them, you can either pass the object directly or pass a ComponentSpec directly.</li></ul> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-1ff9ave">💡 <strong>Component Type Changes</strong>: The component type (pretrained vs config-based) can change when you update components. These types are initially defined in pipeline blocks’ <code>expected_components</code> field using <code>ComponentSpec</code> with <code>default_creation_method</code>. See the <a href="#customizing-guidance-techniques">Customizing Guidance Techniques</a> section for examples of how this works in practice.</p></div> <p data-svelte-h="svelte-4b0omr"><code>ComponentSpec</code> defines how to create or load components and can actually create them using its <code>create()</code> method (for ConfigMixin objects) or <code>load()</code> method (wrapper around <code>from_pretrained()</code>). When a component is loaded with a ComponentSpec, it gets tagged with a unique ID that encodes its creation parameters, allowing you to always extract the original specification using <code>ComponentSpec.from_component()</code>.</p> <p data-svelte-h="svelte-198tnkb">Now let’s look at how to update pretrained components in practice:</p> <p data-svelte-h="svelte-1bc04it">So instead of</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> UNet2DConditionModel | |
| <span class="hljs-keyword">import</span> torch | |
| unet = UNet2DConditionModel.from_pretrained(<span class="hljs-string">"stabilityai/stable-diffusion-xl-base-1.0"</span>, subfolder=<span class="hljs-string">"unet"</span>, variant=<span class="hljs-string">"fp16"</span>, torch_dtype=torch.float16)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-qptszn">You should load your model like this</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> ComponentSpec, UNet2DConditionModel | |
| unet_spec = ComponentSpec(name=<span class="hljs-string">"unet"</span>,type_hint=UNet2DConditionModel, repo=<span class="hljs-string">"stabilityai/stable-diffusion-xl-base-1.0"</span>, subfolder=<span class="hljs-string">"unet"</span>, variant=<span class="hljs-string">"fp16"</span>) | |
| unet2 = unet_spec.load(torch_dtype=torch.float16)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-2fo2fq">The key difference is that the second unet retains its loading specs, so you can extract the spec and recreate the unet:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># component -> spec</span> | |
| <span class="hljs-meta">>>> </span>spec = ComponentSpec.from_component(<span class="hljs-string">"unet"</span>, unet2) | |
| <span class="hljs-meta">>>> </span>spec | |
| ComponentSpec(name=<span class="hljs-string">'unet'</span>, type_hint=<<span class="hljs-keyword">class</span> <span class="hljs-string">'diffusers.models.unets.unet_2d_condition.UNet2DConditionModel'</span>>, description=<span class="hljs-literal">None</span>, config=<span class="hljs-literal">None</span>, repo=<span class="hljs-string">'stabilityai/stable-diffusion-xl-base-1.0'</span>, subfolder=<span class="hljs-string">'unet'</span>, variant=<span class="hljs-string">'fp16'</span>, revision=<span class="hljs-literal">None</span>, default_creation_method=<span class="hljs-string">'from_pretrained'</span>) | |
| <span class="hljs-comment"># spec -> component</span> | |
| <span class="hljs-meta">>>> </span>unet2_recreatd = spec.load(torch_dtype=torch.float16)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1tkcinu">To replace the unet in the pipeline</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->t2i_pipeline.update<span class="hljs-constructor">_components(<span class="hljs-params">unet</span>=<span class="hljs-params">unet2</span>)</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-e1vgvl">Not only is the <code>unet</code> component swapped, but its loading specs are also updated from “RunDiffusion/Juggernaut-XL-v9” to “stabilityai/stable-diffusion-xl-base-1.0” in pipeline config. This means that if you save the pipeline now and load it back with <code>from_pretrained</code>, the new pipeline will by default load the SDXL original unet.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->>>> t2i_pipeline | |
| <span class="hljs-symbol">StableDiffusionXLModularPipeline</span> { | |
| ... | |
| <span class="hljs-string">"unet"</span>: [ | |
| <span class="hljs-string">"diffusers"</span>, | |
| <span class="hljs-string">"UNet2DConditionModel"</span>, | |
| { | |
| <span class="hljs-string">"repo"</span>: <span class="hljs-string">"stabilityai/stable-diffusion-xl-base-1.0"</span>, | |
| <span class="hljs-string">"revision"</span>: null, | |
| <span class="hljs-string">"subfolder"</span>: <span class="hljs-string">"unet"</span>, | |
| <span class="hljs-string">"type_hint"</span>: [ | |
| <span class="hljs-string">"diffusers"</span>, | |
| <span class="hljs-string">"UNet2DConditionModel"</span> | |
| ], | |
| <span class="hljs-string">"variant"</span>: <span class="hljs-string">"fp16"</span> | |
| } | |
| ], | |
| ... | |
| } <!-- HTML_TAG_END --></pre></div> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-104t12l">💡 <strong>Modifying Component Specs</strong>: You can get a copy of the current component spec from the pipeline using <code>get_component_spec()</code>. This makes it easy to modify the spec and updating components.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>unet_spec = t2i_pipeline.get_component_spec(<span class="hljs-string">"unet"</span>) | |
| <span class="hljs-meta">>>> </span>unet_spec | |
| ComponentSpec( | |
| name=<span class="hljs-string">'unet'</span>, | |
| type_hint=<<span class="hljs-keyword">class</span> <span class="hljs-string">'diffusers.models.unets.unet_2d_condition.UNet2DConditionModel'</span>>, | |
| repo=<span class="hljs-string">'RunDiffusion/Juggernaut-XL-v9'</span>, | |
| subfolder=<span class="hljs-string">'unet'</span>, | |
| variant=<span class="hljs-string">'fp16'</span>, | |
| default_creation_method=<span class="hljs-string">'from_pretrained'</span> | |
| ) | |
| <span class="hljs-comment"># Modify the spec to load from a different repository</span> | |
| <span class="hljs-meta">>>> </span>unet_spec.repo = <span class="hljs-string">"stabilityai/stable-diffusion-xl-base-1.0"</span> | |
| <span class="hljs-comment"># Load the component with the modified spec</span> | |
| <span class="hljs-meta">>>> </span>unet = unet_spec.load(torch_dtype=torch.float16)<!-- HTML_TAG_END --></pre></div></div> <h2 class="relative group"><a id="customizing-guidance-techniques" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#customizing-guidance-techniques"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Customizing Guidance Techniques</span></h2> <p data-svelte-h="svelte-13obvjk">Guiders are implementations of different <a href="https://huggingface.co/papers/2207.12598" rel="nofollow">classifier-free guidance</a> techniques that can be applied during the denoising process to improve generation quality, control, and adherence to prompts. They work by steering the model predictions towards desired directions and away from undesired directions. In diffusers, guiders are implemented as subclasses of <code>BaseGuidance</code>. They can easily be integrated into modular pipelines and provide a flexible way to enhance generation quality without modifying the underlying diffusion models.</p> <p data-svelte-h="svelte-1gum3yp"><strong>ClassifierFreeGuidance (CFG)</strong> is the first and most common guidance technique, used in all our standard pipelines. We also offer many other guidance techniques from the latest research in this area - <strong>PerturbedAttentionGuidance (PAG)</strong>, <strong>SkipLayerGuidance (SLG)</strong>, <strong>SmoothedEnergyGuidance (SEG)</strong>, and others that can provide better results for specific use cases.</p> <p data-svelte-h="svelte-p53xgu">This section demonstrates how to use guiders using the component updating methods we just learned. Since <code>BaseGuidance</code> components are stateless (similar to schedulers), they are typically created with default configurations during pipeline initialization using <code>default_creation_method='from_config'</code>. This means they don’t require loading specs from the repository - you won’t see guider listed in <code>modular_model_index.json</code> files.</p> <p data-svelte-h="svelte-dt9m29">Let’s take a look at the default guider configuration:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>t2i_pipeline.get_component_spec(<span class="hljs-string">"guider"</span>) | |
| ComponentSpec(name=<span class="hljs-string">'guider'</span>, type_hint=<<span class="hljs-keyword">class</span> <span class="hljs-string">'diffusers.guiders.classifier_free_guidance.ClassifierFreeGuidance'</span>>, description=<span class="hljs-literal">None</span>, config=FrozenDict([(<span class="hljs-string">'guidance_scale'</span>, <span class="hljs-number">7.5</span>), (<span class="hljs-string">'guidance_rescale'</span>, <span class="hljs-number">0.0</span>), (<span class="hljs-string">'use_original_formulation'</span>, <span class="hljs-literal">False</span>), (<span class="hljs-string">'start'</span>, <span class="hljs-number">0.0</span>), (<span class="hljs-string">'stop'</span>, <span class="hljs-number">1.0</span>), (<span class="hljs-string">'_use_default_values'</span>, [<span class="hljs-string">'start'</span>, <span class="hljs-string">'guidance_rescale'</span>, <span class="hljs-string">'stop'</span>, <span class="hljs-string">'use_original_formulation'</span>])]), repo=<span class="hljs-literal">None</span>, subfolder=<span class="hljs-literal">None</span>, variant=<span class="hljs-literal">None</span>, revision=<span class="hljs-literal">None</span>, default_creation_method=<span class="hljs-string">'from_config'</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1xjugy8">As you can see, the guider is configured to use <code>ClassifierFreeGuidance</code> with default parameters and <code>default_creation_method='from_config'</code>, meaning it’s created during pipeline initialization rather than loaded from a repository. Let’s verify this, here we run <code>init_pipeline()</code> without a modular repo, and there it is, a guider with the default configuration we just saw</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>pipeline = t2i_blocks.init_pipeline() | |
| <span class="hljs-meta">>>> </span>pipeline.guider | |
| ClassifierFreeGuidance { | |
| <span class="hljs-string">"_class_name"</span>: <span class="hljs-string">"ClassifierFreeGuidance"</span>, | |
| <span class="hljs-string">"_diffusers_version"</span>: <span class="hljs-string">"0.35.0.dev0"</span>, | |
| <span class="hljs-string">"guidance_rescale"</span>: <span class="hljs-number">0.0</span>, | |
| <span class="hljs-string">"guidance_scale"</span>: <span class="hljs-number">7.5</span>, | |
| <span class="hljs-string">"start"</span>: <span class="hljs-number">0.0</span>, | |
| <span class="hljs-string">"stop"</span>: <span class="hljs-number">1.0</span>, | |
| <span class="hljs-string">"use_original_formulation"</span>: false | |
| }<!-- HTML_TAG_END --></pre></div> <h4 class="relative group"><a id="modify-parameters-of-the-same-guider-type" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#modify-parameters-of-the-same-guider-type"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Modify Parameters of the Same Guider Type</span></h4> <p data-svelte-h="svelte-1kya8e6">To change parameters of the same guider type (e.g., adjusting the <code>guidance_scale</code> for CFG), you have two options:</p> <p data-svelte-h="svelte-qhjrz4"><strong>Option 1: Use ComponentSpec.create() method</strong></p> <p data-svelte-h="svelte-1yvbgva">You just need to pass the parameter with the new value to override the default one.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>guider_spec = t2i_pipeline.get_component_spec(<span class="hljs-string">"guider"</span>) | |
| <span class="hljs-meta">>>> </span>guider = guider_spec.create(guidance_scale=<span class="hljs-number">10</span>) | |
| <span class="hljs-meta">>>> </span>t2i_pipeline.update_components(guider=guider)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1dpjpjf"><strong>Option 2: Pass ComponentSpec directly</strong></p> <p data-svelte-h="svelte-1bgqadd">Update the spec directly and pass it to <code>update_components()</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>guider_spec = t2i_pipeline.get_component_spec(<span class="hljs-string">"guider"</span>) | |
| <span class="hljs-meta">>>> </span>guider_spec.config[<span class="hljs-string">"guidance_scale"</span>] = <span class="hljs-number">10</span> | |
| <span class="hljs-meta">>>> </span>t2i_pipeline.update_components(guider=guider_spec)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-s5g7hb">Both approaches produce the same result:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>t2i_pipeline.guider | |
| ClassifierFreeGuidance { | |
| <span class="hljs-string">"_class_name"</span>: <span class="hljs-string">"ClassifierFreeGuidance"</span>, | |
| <span class="hljs-string">"_diffusers_version"</span>: <span class="hljs-string">"0.35.0.dev0"</span>, | |
| <span class="hljs-string">"guidance_rescale"</span>: <span class="hljs-number">0.0</span>, | |
| <span class="hljs-string">"guidance_scale"</span>: <span class="hljs-number">10</span>, | |
| <span class="hljs-string">"start"</span>: <span class="hljs-number">0.0</span>, | |
| <span class="hljs-string">"stop"</span>: <span class="hljs-number">1.0</span>, | |
| <span class="hljs-string">"use_original_formulation"</span>: false | |
| }<!-- HTML_TAG_END --></pre></div> <h4 class="relative group"><a id="switch-to-a-different-guider-type" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#switch-to-a-different-guider-type"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Switch to a Different Guider Type</span></h4> <p data-svelte-h="svelte-tulvz4">Switching between guidance techniques is as simple as passing a guider object of that technique:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> LayerSkipConfig, PerturbedAttentionGuidance | |
| config = LayerSkipConfig(indices=[<span class="hljs-number">2</span>, <span class="hljs-number">9</span>], fqn=<span class="hljs-string">"mid_block.attentions.0.transformer_blocks"</span>, skip_attention=<span class="hljs-literal">False</span>, skip_attention_scores=<span class="hljs-literal">True</span>, skip_ff=<span class="hljs-literal">False</span>) | |
| guider = PerturbedAttentionGuidance( | |
| guidance_scale=<span class="hljs-number">5.0</span>, perturbed_guidance_scale=<span class="hljs-number">2.5</span>, perturbed_guidance_config=config | |
| ) | |
| t2i_pipeline.update_components(guider=guider)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-wxznmr">Note that you will get a warning about changing the guider type, which is expected:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-module-access"><span class="hljs-module"><span class="hljs-identifier">ModularPipeline</span>.</span></span>update_components: adding guider <span class="hljs-keyword">with</span> <span class="hljs-keyword">new</span> <span class="hljs-keyword">type</span>: PerturbedAttentionGuidance, previous <span class="hljs-keyword">type</span>: ClassifierFreeGuidance<!-- HTML_TAG_END --></pre></div> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><ul data-svelte-h="svelte-pva5xa"><li>For <code>from_config</code> components (like guiders, schedulers): You can pass an object of required type OR pass a ComponentSpec directly (which calls <code>create()</code> under the hood)</li> <li>For <code>from_pretrained</code> components (like models): You must use ComponentSpec to ensure proper tagging and loading</li></ul></div> <p data-svelte-h="svelte-1x24ygx">Let’s verify that the guider has been updated:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>t2i_pipeline.guider | |
| PerturbedAttentionGuidance { | |
| <span class="hljs-string">"_class_name"</span>: <span class="hljs-string">"PerturbedAttentionGuidance"</span>, | |
| <span class="hljs-string">"_diffusers_version"</span>: <span class="hljs-string">"0.35.0.dev0"</span>, | |
| <span class="hljs-string">"guidance_rescale"</span>: <span class="hljs-number">0.0</span>, | |
| <span class="hljs-string">"guidance_scale"</span>: <span class="hljs-number">5.0</span>, | |
| <span class="hljs-string">"perturbed_guidance_config"</span>: { | |
| <span class="hljs-string">"dropout"</span>: <span class="hljs-number">1.0</span>, | |
| <span class="hljs-string">"fqn"</span>: <span class="hljs-string">"mid_block.attentions.0.transformer_blocks"</span>, | |
| <span class="hljs-string">"indices"</span>: [ | |
| <span class="hljs-number">2</span>, | |
| <span class="hljs-number">9</span> | |
| ], | |
| <span class="hljs-string">"skip_attention"</span>: false, | |
| <span class="hljs-string">"skip_attention_scores"</span>: true, | |
| <span class="hljs-string">"skip_ff"</span>: false | |
| }, | |
| <span class="hljs-string">"perturbed_guidance_layers"</span>: null, | |
| <span class="hljs-string">"perturbed_guidance_scale"</span>: <span class="hljs-number">2.5</span>, | |
| <span class="hljs-string">"perturbed_guidance_start"</span>: <span class="hljs-number">0.01</span>, | |
| <span class="hljs-string">"perturbed_guidance_stop"</span>: <span class="hljs-number">0.2</span>, | |
| <span class="hljs-string">"start"</span>: <span class="hljs-number">0.0</span>, | |
| <span class="hljs-string">"stop"</span>: <span class="hljs-number">1.0</span>, | |
| <span class="hljs-string">"use_original_formulation"</span>: false | |
| } | |
| <!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-169ip08">The component spec has also been updated to reflect the new guider type:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>t2i_pipeline.get_component_spec(<span class="hljs-string">"guider"</span>) | |
| ComponentSpec(name=<span class="hljs-string">'guider'</span>, type_hint=<<span class="hljs-keyword">class</span> <span class="hljs-string">'diffusers.guiders.perturbed_attention_guidance.PerturbedAttentionGuidance'</span>>, description=<span class="hljs-literal">None</span>, config=FrozenDict([(<span class="hljs-string">'guidance_scale'</span>, <span class="hljs-number">5.0</span>), (<span class="hljs-string">'perturbed_guidance_scale'</span>, <span class="hljs-number">2.5</span>), (<span class="hljs-string">'perturbed_guidance_start'</span>, <span class="hljs-number">0.01</span>), (<span class="hljs-string">'perturbed_guidance_stop'</span>, <span class="hljs-number">0.2</span>), (<span class="hljs-string">'perturbed_guidance_layers'</span>, <span class="hljs-literal">None</span>), (<span class="hljs-string">'perturbed_guidance_config'</span>, LayerSkipConfig(indices=[<span class="hljs-number">2</span>, <span class="hljs-number">9</span>], fqn=<span class="hljs-string">'mid_block.attentions.0.transformer_blocks'</span>, skip_attention=<span class="hljs-literal">False</span>, skip_attention_scores=<span class="hljs-literal">True</span>, skip_ff=<span class="hljs-literal">False</span>, dropout=<span class="hljs-number">1.0</span>)), (<span class="hljs-string">'guidance_rescale'</span>, <span class="hljs-number">0.0</span>), (<span class="hljs-string">'use_original_formulation'</span>, <span class="hljs-literal">False</span>), (<span class="hljs-string">'start'</span>, <span class="hljs-number">0.0</span>), (<span class="hljs-string">'stop'</span>, <span class="hljs-number">1.0</span>), (<span class="hljs-string">'_use_default_values'</span>, [<span class="hljs-string">'perturbed_guidance_start'</span>, <span class="hljs-string">'use_original_formulation'</span>, <span class="hljs-string">'perturbed_guidance_layers'</span>, <span class="hljs-string">'stop'</span>, <span class="hljs-string">'start'</span>, <span class="hljs-string">'guidance_rescale'</span>, <span class="hljs-string">'perturbed_guidance_stop'</span>]), (<span class="hljs-string">'_class_name'</span>, <span class="hljs-string">'PerturbedAttentionGuidance'</span>), (<span class="hljs-string">'_diffusers_version'</span>, <span class="hljs-string">'0.35.0.dev0'</span>)]), repo=<span class="hljs-literal">None</span>, subfolder=<span class="hljs-literal">None</span>, variant=<span class="hljs-literal">None</span>, revision=<span class="hljs-literal">None</span>, default_creation_method=<span class="hljs-string">'from_config'</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-c0anvk">The “guider” is still a <code>from_config</code> component: is still not included in the pipeline config and will not be saved into the <code>modular_model_index.json</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">assert</span> <span class="hljs-string">"guider"</span> <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> t2i_pipeline.config<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1nap4cr">However, you can change it to a <code>from_pretrained</code> component, which allows you to upload your customized guider to the Hub and load it into your pipeline.</p> <h4 class="relative group"><a id="loading-custom-guiders-from-hub" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#loading-custom-guiders-from-hub"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Loading Custom Guiders from Hub</span></h4> <p data-svelte-h="svelte-1hqgbsy">If you already have a guider saved on the Hub and a <code>modular_model_index.json</code> with the loading spec for that guider, it will automatically be changed to a <code>from_pretrained</code> component during pipeline initialization.</p> <p data-svelte-h="svelte-1ae1ij0">For example, this <code>modular_model_index.json</code> includes loading specs for the guider:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-punctuation">{</span> | |
| <span class="hljs-attr">"guider"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">[</span> | |
| <span class="hljs-literal"><span class="hljs-keyword">null</span></span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-literal"><span class="hljs-keyword">null</span></span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-punctuation">{</span> | |
| <span class="hljs-attr">"repo"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"YiYiXu/modular-loader-t2i-guider"</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"revision"</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">null</span></span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"subfolder"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"pag_guider"</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"type_hint"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">[</span> | |
| <span class="hljs-string">"diffusers"</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-string">"PerturbedAttentionGuidance"</span> | |
| <span class="hljs-punctuation">]</span><span class="hljs-punctuation">,</span> | |
| <span class="hljs-attr">"variant"</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">null</span></span> | |
| <span class="hljs-punctuation">}</span> | |
| <span class="hljs-punctuation">]</span> | |
| <span class="hljs-punctuation">}</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-16q8ck">When you use this repository to create a pipeline with the same blocks (that originally configured guider as a <code>from_config</code> component), the guider becomes a <code>from_pretrained</code> component. This means it doesn’t get created during initialization, and after you call <code>load_default_components()</code>, it loads based on the spec - resulting in the PAG guider instead of the default CFG.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->t2i_pipeline = t2i_blocks.init_pipeline(<span class="hljs-string">"YiYiXu/modular-doc-guider"</span>) | |
| <span class="hljs-keyword">assert</span> t2i_pipeline.guider <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-comment"># Not created during init</span> | |
| t2i_pipeline.load_default_components() | |
| t2i_pipeline.guider <span class="hljs-comment"># Now loaded as PAG guider</span><!-- HTML_TAG_END --></pre></div> <h4 class="relative group"><a id="upload-custom-guider-to-hub-for-easy-loading--sharing" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#upload-custom-guider-to-hub-for-easy-loading--sharing"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Upload Custom Guider to Hub for Easy Loading & Sharing</span></h4> <p data-svelte-h="svelte-1h04j0a">Now let’s see how we can share the guider on the Hub and change it to a <code>from_pretrained</code> component.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->guider.push_to_hub(<span class="hljs-string">"YiYiXu/modular-loader-t2i-guider"</span>, subfolder=<span class="hljs-string">"pag_guider"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1ory5le">Voilà! Now you have a subfolder called <code>pag_guider</code> on that repository.</p> <p data-svelte-h="svelte-ei0qxu">You have a few options to make this guider available in your pipeline:</p> <ol data-svelte-h="svelte-tmtq35"><li><p><strong>Directly modify the <code>modular_model_index.json</code></strong> to add a loading spec for the guider by pointing to a folder containing the desired guider config.</p></li> <li><p><strong>Use the <code>update_components</code> method</strong> to change it to a <code>from_pretrained</code> component for your pipeline. This is easier if you just want to try it out with different repositories.</p></li></ol> <p data-svelte-h="svelte-kst83c">Let’s use the second approach and change our guider_spec to use <code>from_pretrained</code> as the default creation method and update the loading spec to use this subfolder we just created:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->guider_spec = t2i_pipeline.get_component_spec(<span class="hljs-string">"guider"</span>) | |
| guider_spec.default_creation_method=<span class="hljs-string">"from_pretrained"</span> | |
| guider_spec.repo=<span class="hljs-string">"YiYiXu/modular-loader-t2i-guider"</span> | |
| guider_spec.subfolder=<span class="hljs-string">"pag_guider"</span> | |
| pag_guider = guider_spec.load() | |
| t2i_pipeline.update_components(guider=pag_guider)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1y9qlex">You will get a warning about changing the creation method:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->ModularPipeline.update_components: changing <span class="hljs-keyword">the</span> default_creation_method <span class="hljs-keyword">of</span> guider <span class="hljs-built_in">from</span> from_config <span class="hljs-built_in">to</span> from_pretrained.<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1pt9ert">Now not only the <code>guider</code> component and its component_spec are updated, but so is the pipeline config.</p> <p data-svelte-h="svelte-s37sho">If you want to change the default behavior for future pipelines, you can push the updated pipeline to the Hub. This way, when others use your repository, they’ll get the PAG guider by default. However, this is optional - you don’t have to do this if you just want to experiment locally.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->t2i_pipeline.push_to_hub(<span class="hljs-string">"YiYiXu/modular-doc-guider"</span>)<!-- HTML_TAG_END --></pre></div> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-1esqlay">Experiment with different techniques and parameters to find what works best for your specific use case! You can find all the guider class we support [here](TODO: API doc)</p> <p data-svelte-h="svelte-50g51l">Additionally, you can write your own guider implementations, for example, CFG Zero* combined with Skip Layer Guidance, and they should be compatible out-of-the-box with modular diffusers!</p></div> <h2 class="relative group"><a id="running-a-modularpipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#running-a-modularpipeline"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Running a ModularPipeline</span></h2> <p data-svelte-h="svelte-19vwwbj">The API to run the <code>ModularPipeline</code> is very similar to how you would run a regular <code>DiffusionPipeline</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>image = pipeline(prompt=<span class="hljs-string">"a cat"</span>, num_inference_steps=<span class="hljs-number">15</span>, output=<span class="hljs-string">"images"</span>)[<span class="hljs-number">0</span>]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-13lkbx5">There are a few key differences though:</p> <ol data-svelte-h="svelte-16kbj5a"><li>You can also pass a <code>PipelineState</code> object directly to the pipeline instead of individual arguments</li> <li>If you do not specify the <code>output</code> argument, it returns the <code>PipelineState</code> object</li> <li>You can pass a list as <code>output</code>, e.g. <code>pipeline(... output=["images", "latents"])</code> will return a dictionary containing both the generated image and the final denoised latents</li></ol> <p data-svelte-h="svelte-1k4wrr3">Under the hood, <code>ModularPipeline</code>’s <code>__call__</code> method is a wrapper around the pipeline blocks’ <code>__call__</code> method: it creates a <code>PipelineState</code> object and populates it with user inputs, then returns the output to the user based on the <code>output</code> argument. It also ensures that all pipeline-level config and components are exposed to all pipeline blocks by preparing and passing a <code>components</code> input.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-61h1bw">You can inspect the docstring of a <code>ModularPipeline</code> to check what arguments the pipeline accepts and how to specify the <code>output</code> you want. It will list all available outputs (basically everything in the intermediate pipeline state) so you can choose from the list.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->t2i_pipeline.doc<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-l6qwlc"><strong>Important</strong>: It is important to always check the docstring because arguments can be different from standard pipelines that you’re familar with. For example, in Modular Diffusers we standardized controlnet image input as <code>control_image</code>, but regular pipelines have inconsistencies over the names, e.g. controlnet text-to-image uses <code>image</code> while SDXL controlnet img2img uses <code>control_image</code>.</p> <p data-svelte-h="svelte-8hegdt"><strong>Note</strong>: The <code>output</code> list might be longer than you expected - it includes everything in the intermediate state that you can choose to return. Most of the time, you’ll just want <code>output="images"</code> or <code>output="latents"</code>.</p></div> <h4 class="relative group"><a id="text-to-image-image-to-image-and-inpainting" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#text-to-image-image-to-image-and-inpainting"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Text-to-Image, Image-to-Image, and Inpainting</span></h4> <p data-svelte-h="svelte-119pbdc">These are minimum inference examples for basic tasks: text-to-image, image-to-image, and inpainting. The process to create different pipelines is the same - only difference is the block classes presets. The inference is also more or less same to standard pipelines, but please always check <code>.doc</code> for correct input names and remember to pass <code>output="images"</code>.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">text-to-image </div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">image-to-image </div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">inpainting </div></div> <div class="language-select"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers.modular_pipelines <span class="hljs-keyword">import</span> SequentialPipelineBlocks | |
| <span class="hljs-keyword">from</span> diffusers.modular_pipelines.stable_diffusion_xl <span class="hljs-keyword">import</span> TEXT2IMAGE_BLOCKS | |
| <span class="hljs-comment"># create pipeline from official blocks preset</span> | |
| blocks = SequentialPipelineBlocks.from_blocks_dict(TEXT2IMAGE_BLOCKS) | |
| modular_repo_id = <span class="hljs-string">"YiYiXu/modular-loader-t2i-0704"</span> | |
| pipeline = blocks.init_pipeline(modular_repo_id) | |
| pipeline.load_default_components(torch_dtype=torch.float16) | |
| pipeline.to(<span class="hljs-string">"cuda"</span>) | |
| <span class="hljs-comment"># run pipeline, need to pass a "output=images" argument</span> | |
| image = pipeline(prompt=<span class="hljs-string">"Astronaut in a jungle, cold color palette, muted colors, detailed, 8k"</span>, output=<span class="hljs-string">"images"</span>)[<span class="hljs-number">0</span>] | |
| image.save(<span class="hljs-string">"modular_t2i_out.png"</span>)<!-- HTML_TAG_END --></pre></div> </div> <h4 class="relative group"><a id="controlnet" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#controlnet"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>ControlNet</span></h4> <p data-svelte-h="svelte-1xycv15">For ControlNet, we provide one auto block you can place at the <code>denoise</code> step. Let’s create it and inspect it to see what it tells us.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-1fkro9g">💡 <strong>How to explore new tasks</strong>: When you want to figure out how to do a specific task in Modular Diffusers, it is a good idea to start by checking what block classes presets we offer in <code>ALL_BLOCKS</code>. Then create the block instance and inspect it - it will show you the required components, description, and sub-blocks. This is crucial for understanding what each block does and what it needs.</p></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> diffusers.modular_pipelines.stable_diffusion_xl <span class="hljs-keyword">import</span> ALL_BLOCKS | |
| <span class="hljs-meta">>>> </span>ALL_BLOCKS[<span class="hljs-string">"controlnet"</span>] | |
| InsertableDict([ | |
| <span class="hljs-number">0</span>: (<span class="hljs-string">'denoise'</span>, <<span class="hljs-keyword">class</span> <span class="hljs-string">'diffusers.modular_pipelines.stable_diffusion_xl.modular_blocks.StableDiffusionXLAutoControlnetStep'</span>>) | |
| ]) | |
| <span class="hljs-meta">>>> </span>controlnet_blocks = ALL_BLOCKS[<span class="hljs-string">"controlnet"</span>][<span class="hljs-string">"denoise"</span>]() | |
| <span class="hljs-meta">>>> </span>controlnet_blocks | |
| StableDiffusionXLAutoControlnetStep( | |
| Class: SequentialPipelineBlocks | |
| ==================================================================================================== | |
| This pipeline contains blocks that are selected at runtime based on inputs. | |
| Trigger Inputs: {<span class="hljs-string">'mask'</span>, <span class="hljs-string">'control_mode'</span>, <span class="hljs-string">'control_image'</span>, <span class="hljs-string">'controlnet_cond'</span>} | |
| Use `get_execution_blocks()` <span class="hljs-keyword">with</span> <span class="hljs-built_in">input</span> names to see selected blocks (e.g. `get_execution_blocks(<span class="hljs-string">'mask'</span>)`). | |
| ==================================================================================================== | |
| Description: Controlnet auto step that prepare the controlnet <span class="hljs-built_in">input</span> <span class="hljs-keyword">and</span> denoise the latents. It works <span class="hljs-keyword">for</span> both controlnet <span class="hljs-keyword">and</span> controlnet_union <span class="hljs-keyword">and</span> supports text2img, img2img <span class="hljs-keyword">and</span> inpainting tasks. (it should be replace at <span class="hljs-string">'denoise'</span> step) | |
| Components: | |
| controlnet (`ControlNetUnionModel`) | |
| control_image_processor (`VaeImageProcessor`) | |
| scheduler (`EulerDiscreteScheduler`) | |
| unet (`UNet2DConditionModel`) | |
| guider (`ClassifierFreeGuidance`) | |
| Sub-Blocks: | |
| [<span class="hljs-number">0</span>] controlnet_input (StableDiffusionXLAutoControlNetInputStep) | |
| Description: Controlnet Input step that prepare the controlnet <span class="hljs-built_in">input</span>. | |
| This <span class="hljs-keyword">is</span> an auto pipeline block that works <span class="hljs-keyword">for</span> both controlnet <span class="hljs-keyword">and</span> controlnet_union. | |
| (it should be called right before the denoise step) - `StableDiffusionXLControlNetUnionInputStep` <span class="hljs-keyword">is</span> called to prepare the controlnet <span class="hljs-built_in">input</span> when `control_mode` <span class="hljs-keyword">and</span> `control_image` are provided. | |
| - `StableDiffusionXLControlNetInputStep` <span class="hljs-keyword">is</span> called to prepare the controlnet <span class="hljs-built_in">input</span> when `control_image` <span class="hljs-keyword">is</span> provided. - <span class="hljs-keyword">if</span> neither `control_mode` nor `control_image` <span class="hljs-keyword">is</span> provided, step will be skipped. | |
| [<span class="hljs-number">1</span>] controlnet_denoise (StableDiffusionXLAutoControlNetDenoiseStep) | |
| Description: Denoise step that iteratively denoise the latents <span class="hljs-keyword">with</span> controlnet. This <span class="hljs-keyword">is</span> a auto pipeline block that using controlnet <span class="hljs-keyword">for</span> text2img, img2img <span class="hljs-keyword">and</span> inpainting tasks.This block should <span class="hljs-keyword">not</span> be used without a controlnet_cond <span class="hljs-built_in">input</span> - `StableDiffusionXLInpaintControlNetDenoiseStep` (inpaint_controlnet_denoise) <span class="hljs-keyword">is</span> used when mask <span class="hljs-keyword">is</span> provided. - `StableDiffusionXLControlNetDenoiseStep` (controlnet_denoise) <span class="hljs-keyword">is</span> used when mask <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> provided but controlnet_cond <span class="hljs-keyword">is</span> provided. - If neither mask nor controlnet_cond are provided, step will be skipped. | |
| )<!-- HTML_TAG_END --></pre></div> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-9bz89q">💡 <strong>Auto Blocks</strong>: This is first time we meet a Auto Blocks! <code>AutoPipelineBlocks</code> automatically adapt to your inputs by combining multiple workflows with conditional logic. This is why one convenient block can work for all tasks and controlnet types. See the <a href="./auto_pipeline_blocks.md">Auto Blocks Guide</a> for more details.</p></div> <p data-svelte-h="svelte-1i8x4c3">The block shows us it has two steps (prepare inputs + denoise) and supports all tasks with both controlnet and controlnet union. Most importantly, it tells us to place it at the ‘denoise’ step. Let’s do exactly that:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers.modular_pipelines <span class="hljs-keyword">import</span> SequentialPipelineBlocks | |
| <span class="hljs-keyword">from</span> diffusers.modular_pipelines.stable_diffusion_xl <span class="hljs-keyword">import</span> TEXT2IMAGE_BLOCKS, StableDiffusionXLAutoControlnetStep | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image | |
| <span class="hljs-comment"># create pipeline from blocks preset</span> | |
| blocks = SequentialPipelineBlocks.from_blocks_dict(TEXT2IMAGE_BLOCKS) | |
| <span class="hljs-comment"># these two lines applies controlnet</span> | |
| controlnet_blocks = StableDiffusionXLAutoControlnetStep() | |
| blocks.sub_blocks[<span class="hljs-string">"denoise"</span>] = controlnet_blocks <!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-zqvktu">Before we convert the blocks into a pipeline and load its components, let’s inspect the blocks and its docs again to make sure it was assembled correctly. You should be able to see that <code>controlnet</code> and <code>control_image_processor</code> are now listed as <code>Components</code>, so we should initialize the pipeline with a repo that contains desired loading specs for these 2 components.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># make sure to a modular_repo including controlnet</span> | |
| modular_repo_id = <span class="hljs-string">"YiYiXu/modular-demo-auto"</span> | |
| pipeline = blocks.init_pipeline(modular_repo_id) | |
| pipeline.load_default_components(torch_dtype=torch.float16) | |
| pipeline.to(<span class="hljs-string">"cuda"</span>) | |
| <span class="hljs-comment"># generate</span> | |
| canny_image = load_image( | |
| <span class="hljs-string">"https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png"</span> | |
| ) | |
| image = pipeline( | |
| prompt=<span class="hljs-string">"a bird"</span>, controlnet_conditioning_scale=<span class="hljs-number">0.5</span>, control_image=canny_image, output=<span class="hljs-string">"images"</span> | |
| )[<span class="hljs-number">0</span>] | |
| image.save(<span class="hljs-string">"modular_control_out.png"</span>)<!-- HTML_TAG_END --></pre></div> <h4 class="relative group"><a id="ip-adapter" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#ip-adapter"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>IP-Adapter</span></h4> <p data-svelte-h="svelte-1gbw09g"><strong>Challenge time!</strong> Before we show you how to apply IP-adapter, try doing it yourself! Use the same process we just walked you through with ControlNet: check the official blocks preset, inspect the block instance and docstring <code>.doc</code>, and adapt a regular IP-adapter example to modular.</p> <p data-svelte-h="svelte-pu1b97">Let’s walk through the steps:</p> <ol data-svelte-h="svelte-1sy384"><li>Check blocks preset</li></ol> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> diffusers.modular_pipelines.stable_diffusion_xl <span class="hljs-keyword">import</span> ALL_BLOCKS | |
| <span class="hljs-meta">>>> </span>ALL_BLOCKS[<span class="hljs-string">"ip_adapter"</span>] | |
| InsertableDict([ | |
| <span class="hljs-number">0</span>: (<span class="hljs-string">'ip_adapter'</span>, <<span class="hljs-keyword">class</span> <span class="hljs-string">'diffusers.modular_pipelines.stable_diffusion_xl.modular_blocks.StableDiffusionXLAutoIPAdapterStep'</span>>) | |
| ])<!-- HTML_TAG_END --></pre></div> <ol start="2" data-svelte-h="svelte-12nwn7m"><li>inspect the block & doc</li></ol> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->>>> <span class="hljs-keyword">from</span> diffusers.modular_pipelines.stable_diffusion_xl <span class="hljs-keyword">import</span> StableDiffusionXLAutoIPAdapterStep | |
| >>> ip_adapter_blocks = StableDiffusionXLAutoIPAdapterStep() | |
| >>> ip_adapter_blocks | |
| StableDiffusionXLAutoIPAdapterStep( | |
| <span class="hljs-keyword">Class</span>: AutoPipelineBlocks | |
| ==================================================================================================== | |
| <span class="hljs-keyword">This</span> pipeline contains blocks that are selected at <span class="hljs-keyword">runtime</span> based on inputs. | |
| Trigger Inputs: {<span class="hljs-string">'ip_adapter_image'</span>} | |
| Use `get_execution_blocks()` with input names to see selected blocks (e.g. `get_execution_blocks(<span class="hljs-string">'ip_adapter_image'</span>)`). | |
| ==================================================================================================== | |
| <span class="hljs-keyword">Description</span>: Run IP Adapter <span class="hljs-keyword">step</span> <span class="hljs-keyword">if</span> `ip_adapter_image` is provided. <span class="hljs-keyword">This</span> <span class="hljs-keyword">step</span> should be placed before the <span class="hljs-string">'input'</span> <span class="hljs-keyword">step</span>. | |
| Components: | |
| image_encoder (`CLIPVisionModelWithProjection`) | |
| feature_extractor (`CLIPImageProcessor`) | |
| unet (`UNet2DConditionModel`) | |
| guider (`ClassifierFreeGuidance`) | |
| Sub-Blocks: | |
| • ip_adapter [trigger: ip_adapter_image] (StableDiffusionXLIPAdapterStep) | |
| <span class="hljs-keyword">Description</span>: IP Adapter <span class="hljs-keyword">step</span> that prepares ip adapter image embeddings. | |
| Note that <span class="hljs-keyword">this</span> <span class="hljs-keyword">step</span> only prepares the embeddings - in order <span class="hljs-keyword">for</span> it to work correctly, you need to load ip adapter weights <span class="hljs-keyword">into</span> unet via ModularPipeline.load_ip_adapter() and pipeline.set_ip_adapter_scale(). | |
| See [ModularIPAdapterMixin](https:<span class="hljs-comment">//huggingface.co/docs/diffusers/api/loaders/ip_adapter#diffusers.loaders.ModularIPAdapterMixin) for more details</span> | |
| )<!-- HTML_TAG_END --></pre></div> <ol start="3" data-svelte-h="svelte-hjvbv0"><li>follow the instruction to build</li></ol> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers.modular_pipelines <span class="hljs-keyword">import</span> SequentialPipelineBlocks | |
| <span class="hljs-keyword">from</span> diffusers.modular_pipelines.stable_diffusion_xl <span class="hljs-keyword">import</span> TEXT2IMAGE_BLOCKS | |
| <span class="hljs-comment"># create pipeline from official blocks preset</span> | |
| blocks = SequentialPipelineBlocks.from_blocks_dict(TEXT2IMAGE_BLOCKS) | |
| <span class="hljs-comment"># insert ip_adapter_blocks before the input step as instructed</span> | |
| blocks.sub_blocks.insert(<span class="hljs-string">"ip_adapter"</span>, ip_adapter_blocks, <span class="hljs-number">1</span>) | |
| <span class="hljs-comment"># inspec the blocks before you convert it into pipelines,</span> | |
| <span class="hljs-comment"># and make sure to use a repo that contains the loading spec for all components</span> | |
| <span class="hljs-comment"># for ip-adapter, you need image_encoder & feature_extractor</span> | |
| modular_repo_id = <span class="hljs-string">"YiYiXu/modular-demo-auto"</span> | |
| pipeline = blocks.init_pipeline(modular_repo_id) | |
| pipeline.load_default_components(torch_dtype=torch.float16) | |
| pipeline.load_ip_adapter( | |
| <span class="hljs-string">"h94/IP-Adapter"</span>, | |
| subfolder=<span class="hljs-string">"sdxl_models"</span>, | |
| weight_name=<span class="hljs-string">"ip-adapter_sdxl.bin"</span> | |
| ) | |
| pipeline.set_ip_adapter_scale(<span class="hljs-number">0.8</span>) | |
| pipeline.to(<span class="hljs-string">"cuda"</span>)<!-- HTML_TAG_END --></pre></div> <ol start="4" data-svelte-h="svelte-f83e52"><li>adapt an example to modular</li></ol> <p data-svelte-h="svelte-af1nse">We are using <a href="https://huggingface.co/docs/diffusers/using-diffusers/ip_adapter?ipadapter-variants=IP-Adapter+Plus#ip-adapter" rel="nofollow">this one</a> from our IP-Adapter doc!</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image | |
| image = load_image(<span class="hljs-string">"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/ip_adapter_diner.png"</span>) | |
| image = pipeline( | |
| prompt=<span class="hljs-string">"a polar bear sitting in a chair drinking a milkshake"</span>, | |
| ip_adapter_image=image, | |
| negative_prompt=<span class="hljs-string">"deformed, ugly, wrong proportion, low res, bad anatomy, worst quality, low quality"</span>, | |
| output=<span class="hljs-string">"images"</span> | |
| )[<span class="hljs-number">0</span>] | |
| image.save(<span class="hljs-string">"modular_ipa_out.png"</span>)<!-- HTML_TAG_END --></pre></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/modular_diffusers/modular_pipeline.md" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span data-svelte-h="svelte-x0xyl0">></span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p> | |
| <script> | |
| { | |
| __sveltekit_9fqn6m = { | |
| assets: "/docs/diffusers/pr_11797/en", | |
| base: "/docs/diffusers/pr_11797/en", | |
| env: {} | |
| }; | |
| const element = document.currentScript.parentElement; | |
| const data = [null,null]; | |
| Promise.all([ | |
| import("/docs/diffusers/pr_11797/en/_app/immutable/entry/start.2e1ee9a7.js"), | |
| import("/docs/diffusers/pr_11797/en/_app/immutable/entry/app.87c95e21.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 232], | |
| data, | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 194 kB
- Xet hash:
- 253b8f851a4c3345146700e2df54e0510c085a97fc75ea8c31ccba6becc20c1a
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.