Buckets:

hf-doc-build
/

doc-dev

hf-doc-build/doc-dev / diffusers /pr_11686 /en /modular_diffusers /end_to_end_guide.html

96.7 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"End-to-End Developer Guide: Building with Modular Diffusers","local":"end-to-end-developer-guide-building-with-modular-diffusers","sections":[{"title":"Differential Diffusion Pipeline","local":"differential-diffusion-pipeline","sections":[{"title":"Start with an existing pipeline","local":"start-with-an-existing-pipeline","sections":[],"depth":3},{"title":"Build a Working Pipeline Structure","local":"build-a-working-pipeline-structure","sections":[],"depth":3},{"title":"Set up an example","local":"set-up-an-example","sections":[],"depth":3},{"title":"implement your custom logic and test incrementally","local":"implement-your-custom-logic-and-test-incrementally","sections":[],"depth":3}],"depth":2},{"title":"Adding IP-adapter","local":"adding-ip-adapter","sections":[],"depth":2},{"title":"Working with ControlNets","local":"working-with-controlnets","sections":[],"depth":2},{"title":"Creating a Modular Repo","local":"creating-a-modular-repo","sections":[],"depth":2},{"title":"deploy a mellon node","local":"deploy-a-mellon-node","sections":[],"depth":2}],"depth":1}">
	<link href="/docs/diffusers/pr_11686/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/entry/start.2b9667fb.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/chunks/scheduler.8c3d61f6.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/chunks/singletons.756349ae.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/chunks/index.0997d446.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/chunks/paths.8d5937da.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/entry/app.a2a6117e.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/chunks/index.da70eac4.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/nodes/0.a31d0923.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/nodes/224.b0ad2791.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/chunks/Tip.1d9b8c37.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/chunks/CodeBlock.a9c4becf.js">
	<link rel="modulepreload" href="/docs/diffusers/pr_11686/en/_app/immutable/chunks/getInferenceSnippets.d00e08ac.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"End-to-End Developer Guide: Building with Modular Diffusers","local":"end-to-end-developer-guide-building-with-modular-diffusers","sections":[{"title":"Differential Diffusion Pipeline","local":"differential-diffusion-pipeline","sections":[{"title":"Start with an existing pipeline","local":"start-with-an-existing-pipeline","sections":[],"depth":3},{"title":"Build a Working Pipeline Structure","local":"build-a-working-pipeline-structure","sections":[],"depth":3},{"title":"Set up an example","local":"set-up-an-example","sections":[],"depth":3},{"title":"implement your custom logic and test incrementally","local":"implement-your-custom-logic-and-test-incrementally","sections":[],"depth":3}],"depth":2},{"title":"Adding IP-adapter","local":"adding-ip-adapter","sections":[],"depth":2},{"title":"Working with ControlNets","local":"working-with-controlnets","sections":[],"depth":2},{"title":"Creating a Modular Repo","local":"creating-a-modular-repo","sections":[],"depth":2},{"title":"deploy a mellon node","local":"deploy-a-mellon-node","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="end-to-end-developer-guide-building-with-modular-diffusers" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#end-to-end-developer-guide-building-with-modular-diffusers"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>End-to-End Developer Guide: Building with Modular Diffusers</span></h1> <div class="course-tip course-tip-orange bg-gradient-to-br dark:bg-gradient-to-r before:border-orange-500 dark:before:border-orange-800 from-orange-50 dark:from-gray-900 to-white dark:to-gray-950 border border-orange-50 text-orange-700 dark:text-gray-400"><p data-svelte-h="svelte-11qh5l3">🧪 <strong>Experimental Feature</strong>: Modular Diffusers is an experimental feature we are actively developing. The API may be subject to breaking changes.</p></div> <p data-svelte-h="svelte-ah51rm">In this tutorial we will walk through the process of adding a new pipeline to the modular framework using differential diffusion as our example. We’ll cover the complete workflow from implementation to deployment: implementing the new pipeline, ensuring compatibility with existing tools, sharing the code on Hugging Face Hub, and deploying it as a UI node.</p> <p data-svelte-h="svelte-q8pbl8">We’ll also demonstrate the 4-step framework process we use for implementing new basic pipelines in the modular system.</p> <ol data-svelte-h="svelte-ic8w5h"><li><p><strong>Start with an existing pipeline as a base</strong></p> <ul><li>Identify which existing pipeline is most similar to the one you want to implement</li> <li>Determine what part of the pipeline needs modification</li></ul></li> <li><p><strong>Build a working pipeline structure first</strong></p> <ul><li>Assemble the complete pipeline structure</li> <li>Use existing blocks wherever possible</li> <li>For new blocks, create placeholders (e.g. you can copy from similar blocks and change the name) without implementing custom logic just yet</li></ul></li> <li><p><strong>Set up an example</strong></p> <ul><li>Create a simple inference script with expected inputs/outputs</li></ul></li> <li><p><strong>Implement your custom logic and test incrementally</strong></p> <ul><li>Add the custom logics the blocks you want to change</li> <li>Test incrementally, and inspect pipeline states and debug as needed</li></ul></li></ol> <p data-svelte-h="svelte-vd2vm7">Let’s see how this works with the Differential Diffusion example.</p> <h2 class="relative group"><a id="differential-diffusion-pipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#differential-diffusion-pipeline"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Differential Diffusion Pipeline</span></h2> <h3 class="relative group"><a id="start-with-an-existing-pipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#start-with-an-existing-pipeline"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Start with an existing pipeline</span></h3> <p data-svelte-h="svelte-1ckzp7j">Differential diffusion (<a href="https://differential-diffusion.github.io/" rel="nofollow">https://differential-diffusion.github.io/</a>) is an image-to-image workflow, so it makes sense for us to start with the preset of pipeline blocks used to build img2img pipeline (<code>IMAGE2IMAGE_BLOCKS</code>) and see how we can build this new pipeline with them.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> diffusers.modular_pipelines.stable_diffusion_xl <span class="hljs-keyword">import</span> IMAGE2IMAGE_BLOCKS
	<span class="hljs-meta">>>> </span>IMAGE2IMAGE_BLOCKS = InsertableDict([
	<span class="hljs-meta">... </span> (<span class="hljs-string">"text_encoder"</span>, StableDiffusionXLTextEncoderStep),
	<span class="hljs-meta">... </span> (<span class="hljs-string">"image_encoder"</span>, StableDiffusionXLVaeEncoderStep),
	<span class="hljs-meta">... </span> (<span class="hljs-string">"input"</span>, StableDiffusionXLInputStep),
	<span class="hljs-meta">... </span> (<span class="hljs-string">"set_timesteps"</span>, StableDiffusionXLImg2ImgSetTimestepsStep),
	<span class="hljs-meta">... </span> (<span class="hljs-string">"prepare_latents"</span>, StableDiffusionXLImg2ImgPrepareLatentsStep),
	<span class="hljs-meta">... </span> (<span class="hljs-string">"prepare_add_cond"</span>, StableDiffusionXLImg2ImgPrepareAdditionalConditioningStep),
	<span class="hljs-meta">... </span> (<span class="hljs-string">"denoise"</span>, StableDiffusionXLDenoiseStep),
	<span class="hljs-meta">... </span> (<span class="hljs-string">"decode"</span>, StableDiffusionXLDecodeStep)
	<span class="hljs-meta">... </span>])<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-16iflyl">Note that “denoise” (<code>StableDiffusionXLDenoiseStep</code>) is a <code>LoopSequentialPipelineBlocks</code> that contains 3 loop blocks (more on LoopSequentialPipelineBlocks <a href="https://huggingface.co/docs/diffusers/modular_diffusers/write_own_pipeline_block#loopsequentialpipelineblocks" rel="nofollow">here</a>)</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>denoise_blocks = IMAGE2IMAGE_BLOCKS[<span class="hljs-string">"denoise"</span>]()
	<span class="hljs-meta">>>> </span><span class="hljs-built_in">print</span>(denoise_blocks)<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->StableDiffusionXLDenoiseStep(
	<span class="hljs-title"> Class:</span> StableDiffusionXLDenoiseLoopWrapper

	<span class="hljs-title"> Description:</span> Denoise step that iteratively denoise the latents.
	Its loop logic is defined in `StableDiffusionXLDenoiseLoopWrapper.__call__` method
	<span class="hljs-title"> At each iteration, it runs blocks defined in `sub_blocks` sequencially:</span>
	- `StableDiffusionXLLoopBeforeDenoiser`
	- `StableDiffusionXLLoopDenoiser`
	- `StableDiffusionXLLoopAfterDenoiser`
	This block supports both text2img <span class="hljs-literal">and</span> img2img tasks.


	<span class="hljs-title"> Components:</span>
	scheduler (`EulerDiscreteScheduler`)
	guider (`ClassifierFreeGuidance`)
	unet (`UNet2DConditionModel`)

	<span class="hljs-title"> Sub-Blocks:</span>
	[<span class="hljs-number">0</span>] before_denoiser (StableDiffusionXLLoopBeforeDenoiser)
	<span class="hljs-title"> Description:</span> step within the denoising loop that prepare the latent input for the denoiser. This block should be used to compose the `sub_blocks` attribute of a `LoopSequentialPipelineBlocks` object (e.g. `StableDiffusionXLDenoiseLoopWrapper`)

	[<span class="hljs-number">1</span>] denoiser (StableDiffusionXLLoopDenoiser)
	<span class="hljs-title"> Description:</span> Step within the denoising loop that denoise the latents with guidance. This block should be used to compose the `sub_blocks` attribute of a `LoopSequentialPipelineBlocks` object (e.g. `StableDiffusionXLDenoiseLoopWrapper`)

	[<span class="hljs-number">2</span>] after_denoiser (StableDiffusionXLLoopAfterDenoiser)
	<span class="hljs-title"> Description:</span> step within the denoising loop that update the latents. This block should be used to compose the `sub_blocks` attribute of a `LoopSequentialPipelineBlocks` object (e.g. `StableDiffusionXLDenoiseLoopWrapper`)

	)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1jaxdnk">Let’s compare standard image-to-image and differential diffusion! The key difference in algorithm is that standard image-to-image diffusion applies uniform noise across all pixels based on a single <code>strength</code> parameter, but differential diffusion uses a change map where each pixel value determines when that region starts denoising. Regions with lower values get “frozen” earlier by replacing them with noised original latents, preserving more of the original image.</p> <p data-svelte-h="svelte-16j8f35">Therefore, the key differences when it comes to pipeline implementation would be:</p> <ol data-svelte-h="svelte-9mzypu"><li>The <code>prepare_latents</code> step (which prepares the change map and pre-computes noised latents for all timesteps)</li> <li>The <code>denoise</code> step (which selectively applies denoising based on the change map)</li> <li>Since differential diffusion doesn’t use the <code>strength</code> parameter, we’ll use the text-to-image <code>set_timesteps</code> step instead of the image-to-image version</li></ol> <p data-svelte-h="svelte-1njl7f7">To implement differntial diffusion, we can reuse most blocks from image-to-image and text-to-image workflows, only modifying the <code>prepare_latents</code> step and the first part of the <code>denoise</code> step (i.e. <code>before_denoiser (StableDiffusionXLLoopBeforeDenoiser)</code>).</p> <p data-svelte-h="svelte-6yjvng">Here’s a flowchart showing the pipeline structure and the changes we need to make:</p> <p data-svelte-h="svelte-19xrt2z"><img src="https://mermaid.ink/img/pako:eNqVVO9r4kAQ_VeWLQWFKEk00eRDwZpa7Q-ucPfpYpE1mdWlcTdsVmpb-7_fZk1tTCl3J0Sy8968N5kZ9g0nIgUc4pUk-Rr9iuYc6d_Ibs14vlXoQYpNrtqo07lAo1jBTi2AlynysWIa6DJmG7KCBnZpsHHMSqkqNjaxKC5ALRTbQKEgLyosMthVnEvIiYRFRhRwVaBoNpmUT0W7MrTJkUbSdJEInlbwxMDXcQpcsAKq6OH_2mDTODIY4yt0J0ReUaYGnLXiJVChdSsB-enfPhBnhnjT-rCQj-1K_8Ygt62YUAVy8Ykf4FvU6XYu9rpuIGqPpvXSzs_RVEj2KrgiGUp02zNQTHBEM_FcK3BfQbBHd7qAst-PxvW-9WOrypnNylG0G9oRUMYBFeolg-IQTTJSFDqOUkZp-fwsQURZloVnlPpLf2kVSoonCM-SwCUuqY6dZ5aqddjLd1YiMiFLNrWorrxj9EOmP4El37lsl_9p5PzFqIqwVwgdN981fDM94bphH5I06R8NXZ_4QcPQPTFs6JltPrS6JssFhw9N817l27bdyM-lSKAo6iVBAAnQY0n9wLO9wbcluY7ruUFDtdguH74K0yENKDkK-8nAG6TfNrfy_bf-HjdrlOfZS7VYSAlU5JAwyhLE9WrWVw1dWdPTXauDsy8LUkdHtnX_pfMnBOvSGluRNbGurbuTHtdZN9Zts1MljC19_7EUh0puwcIbkBtSHvFbic6xWsMG5jjUrymRT3M85-86Jyf8txCbjzQptqs1DinJCn3a5qm-viJG9M26OUYlcH0_jsWWKxwGttHA4Rve4dD1el3H8_yh49hD3_X7roVfcNhx-l3b14PxvGHQ0xMa9t4t_Gp8na7tDvu-4w08HXecweD9D4X54ZI" alt="DiffDiff Pipeline Structure"></p> <h3 class="relative group"><a id="build-a-working-pipeline-structure" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#build-a-working-pipeline-structure"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Build a Working Pipeline Structure</span></h3> <p data-svelte-h="svelte-htmxyh">ok now we’ve identified the blocks to modify, let’s build the pipeline skeleton first - at this stage, our goal is to get the pipeline struture working end-to-end (even though it’s just doing the img2img behavior). I would simply create placeholder blocks by copying from existing ones:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-comment"># Copy existing blocks as placeholders</span>
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">class</span> <span class="hljs-title class_">SDXLDiffDiffPrepareLatentsStep</span>(<span class="hljs-title class_ inherited__">PipelineBlock</span>):
	<span class="hljs-meta">... </span> <span class="hljs-string">"""Copied from StableDiffusionXLImg2ImgPrepareLatentsStep - will modify later"""</span>
	<span class="hljs-meta">... </span> <span class="hljs-comment"># ... same implementation as StableDiffusionXLImg2ImgPrepareLatentsStep</span>
	<span class="hljs-meta">... </span>
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">class</span> <span class="hljs-title class_">SDXLDiffDiffLoopBeforeDenoiser</span>(<span class="hljs-title class_ inherited__">PipelineBlock</span>):
	<span class="hljs-meta">... </span> <span class="hljs-string">"""Copied from StableDiffusionXLLoopBeforeDenoiser - will modify later"""</span>
	<span class="hljs-meta">... </span> <span class="hljs-comment"># ... same implementation as StableDiffusionXLLoopBeforeDenoiser</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1h9rp65"><code>SDXLDiffDiffLoopBeforeDenoiser</code> is the be part of the denoise loop we need to change. Let’s use it to assemble a <code>SDXLDiffDiffDenoiseStep</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">class</span> <span class="hljs-title class_">SDXLDiffDiffDenoiseStep</span>(<span class="hljs-title class_ inherited__">StableDiffusionXLDenoiseLoopWrapper</span>):
	<span class="hljs-meta">... </span> block_classes = [SDXLDiffDiffLoopBeforeDenoiser, StableDiffusionXLLoopDenoiser, StableDiffusionXLLoopAfterDenoiser]
	<span class="hljs-meta">... </span> block_names = [<span class="hljs-string">"before_denoiser"</span>, <span class="hljs-string">"denoiser"</span>, <span class="hljs-string">"after_denoiser"</span>]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1kjlg3r">Now we can put together our differential diffusion pipeline.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>DIFFDIFF_BLOCKS = IMAGE2IMAGE_BLOCKS.copy()
	<span class="hljs-meta">>>> </span>DIFFDIFF_BLOCKS[<span class="hljs-string">"set_timesteps"</span>] = TEXT2IMAGE_BLOCKS[<span class="hljs-string">"set_timesteps"</span>]
	<span class="hljs-meta">>>> </span>DIFFDIFF_BLOCKS[<span class="hljs-string">"prepare_latents"</span>] = SDXLDiffDiffPrepareLatentsStep
	<span class="hljs-meta">>>> </span>DIFFDIFF_BLOCKS[<span class="hljs-string">"denoise"</span>] = SDXLDiffDiffDenoiseStep
	<span class="hljs-meta">>>> </span>
	<span class="hljs-meta">>>> </span>dd_blocks = SequentialPipelineBlocks.from_blocks_dict(DIFFDIFF_BLOCKS)
	<span class="hljs-meta">>>> </span><span class="hljs-built_in">print</span>(dd_blocks)
	<span class="hljs-meta">>>> </span><span class="hljs-comment"># At this point, the pipeline works exactly like img2img since our blocks are just copies</span><!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="set-up-an-example" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#set-up-an-example"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Set up an example</span></h3> <p data-svelte-h="svelte-15obvlk">ok, so now our blocks should be able to compile without an error, we can move on to the next step. Let’s setup a simple example so we can run the pipeline as we build it. diff-diff use same model checkpoints as SDXL so we can fetch the models from a regular SDXL repo.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>dd_pipeline = dd_blocks.init_pipeline(<span class="hljs-string">"YiYiXu/modular-demo-auto"</span>, collection=<span class="hljs-string">"diffdiff"</span>)
	<span class="hljs-meta">>>> </span>dd_pipeline.load_default_componenets(torch_dtype=torch.float16)
	<span class="hljs-meta">>>> </span>dd_pipeline.to(<span class="hljs-string">"cuda"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-dx1yg">We will use this example script:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>image = load_image(<span class="hljs-string">"https://huggingface.co/datasets/OzzyGT/testing-resources/resolve/main/differential/20240329211129_4024911930.png?download=true"</span>)
	<span class="hljs-meta">>>> </span>mask = load_image(<span class="hljs-string">"https://huggingface.co/datasets/OzzyGT/testing-resources/resolve/main/differential/gradient_mask.png?download=true"</span>)
	<span class="hljs-meta">>>> </span>
	<span class="hljs-meta">>>> </span>prompt = <span class="hljs-string">"a green pear"</span>
	<span class="hljs-meta">>>> </span>negative_prompt = <span class="hljs-string">"blurry"</span>
	<span class="hljs-meta">>>> </span>
	<span class="hljs-meta">>>> </span>image = dd_pipeline(
	<span class="hljs-meta">... </span> prompt=prompt,
	<span class="hljs-meta">... </span> negative_prompt=negative_prompt,
	<span class="hljs-meta">... </span> num_inference_steps=<span class="hljs-number">25</span>,
	<span class="hljs-meta">... </span> diffdiff_map=mask,
	<span class="hljs-meta">... </span> image=image,
	<span class="hljs-meta">... </span> output=<span class="hljs-string">"images"</span>
	<span class="hljs-meta">... </span>)[<span class="hljs-number">0</span>]
	<span class="hljs-meta">>>> </span>
	<span class="hljs-meta">>>> </span>image.save(<span class="hljs-string">"diffdiff_out.png"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-11o6vb9">If you run the script right now, you will get a complaint about unexpected input <code>diffdiff_map</code>.
	and you would get the same result as the original img2img pipeline.</p> <h3 class="relative group"><a id="implement-your-custom-logic-and-test-incrementally" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#implement-your-custom-logic-and-test-incrementally"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>implement your custom logic and test incrementally</span></h3> <p data-svelte-h="svelte-t8s3od">Let’s modify the pipeline so that we can get expected result with this example script.</p> <p data-svelte-h="svelte-me5buj">We’ll start with the <code>prepare_latents</code> step. The main changes are:</p> <ul data-svelte-h="svelte-zyaxw1"><li>Requires a new user input <code>diffdiff_map</code></li> <li>Requires new component <code>mask_processor</code> to process the <code>diffdiff_map</code></li> <li>Requires new intermediate inputs: <ul><li>Need <code>timestep</code> instead of <code>latent_timestep</code> to precompute all the latents</li> <li>Need <code>num_inference_steps</code> to create the <code>diffdiff_masks</code></li></ul></li> <li>create a new output <code>diffdiff_masks</code> and <code>original_latents</code></li></ul> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-1fmrtdn">💡 use <code>print(dd_pipeline.doc)</code> to check compiled inputs and outputs of the built piepline.</p> <p data-svelte-h="svelte-16ri2fv">e.g. after we added <code>diffdiff_map</code> as an input in this step, we can run <code>print(dd_pipeline.doc)</code> to verify that it shows up in the docstring as a user input.</p></div> <p data-svelte-h="svelte-11q7ywa">Once we make sure all the variables we need are available in the block state, we can implement the diff-diff logic inside <code>__call__</code>. We created 2 new variables: the change map <code>diffdiff_mask</code> and the pre-computed noised latents for all timesteps <code>original_latents</code>.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-1ktq700">💡 Implement incrementally! Run the example script as you go, and insert <code>print(state)</code> and <code>print(block_state)</code> everywhere inside the <code>__call__</code> method to inspect the intermediate results. This helps you understand what’s going on and what each line you just added does.</p></div> <p data-svelte-h="svelte-8lo2o8">Here are the key changes we made to implement differential diffusion:</p> <p data-svelte-h="svelte-192zqht"><strong>1. Modified <code>prepare_latents</code> step:</strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->class SDXLDiffDiffPrepareLatentsStep(PipelineBlock):
	@property
	def expected_components(self) -> List[ComponentSpec]:
	return [
	ComponentSpec("vae", AutoencoderKL),
	ComponentSpec("scheduler", EulerDiscreteScheduler),
	<span class="hljs-addition">+ ComponentSpec("mask_processor", VaeImageProcessor, config=FrozenDict({"do_normalize": False, "do_convert_grayscale": True}))</span>
	]

	@property
	def inputs(self) -> List[Tuple[str, Any]]:
	return [
	<span class="hljs-addition">+ InputParam("diffdiff_map", required=True),</span>
	]

	@property
	def intermediate_inputs(self) -> List[InputParam]:
	return [
	InputParam("generator"),
	<span class="hljs-deletion">- InputParam("latent_timestep", required=True, type_hint=torch.Tensor),</span>
	<span class="hljs-addition">+ InputParam("timesteps", type_hint=torch.Tensor),</span>
	<span class="hljs-addition">+ InputParam("num_inference_steps", type_hint=int),</span>
	]

	@property
	def intermediate_outputs(self) -> List[OutputParam]:
	return [
	<span class="hljs-addition">+ OutputParam("original_latents", type_hint=torch.Tensor),</span>
	<span class="hljs-addition">+ OutputParam("diffdiff_masks", type_hint=torch.Tensor),</span>
	]

	def __call__(self, components, state: PipelineState):
	# ... existing logic ...
	<span class="hljs-addition">+ # Process change map and create masks</span>
	<span class="hljs-addition">+ diffdiff_map = components.mask_processor.preprocess(block_state.diffdiff_map, height=latent_height, width=latent_width)</span>
	<span class="hljs-addition">+ thresholds = torch.arange(block_state.num_inference_steps, dtype=diffdiff_map.dtype) / block_state.num_inference_steps</span>
	<span class="hljs-addition">+ block_state.diffdiff_masks = diffdiff_map > (thresholds + (block_state.denoising_start or 0))</span>
	<span class="hljs-addition">+ block_state.original_latents = block_state.latents</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1gzuzik"><strong>2. Modified <code>before_denoiser</code> step:</strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->class SDXLDiffDiffLoopBeforeDenoiser(PipelineBlock):
	@property
	def description(self) -> str:
	return (
	"Step within the denoising loop for differential diffusion that prepare the latent input for the denoiser"
	)

	<span class="hljs-addition">+ @property</span>
	<span class="hljs-addition">+ def inputs(self) -> List[Tuple[str, Any]]:</span>
	<span class="hljs-addition">+ return [</span>
	<span class="hljs-addition">+ InputParam("denoising_start"),</span>
	<span class="hljs-addition">+ ]</span>

	@property
	def intermediate_inputs(self) -> List[str]:
	return [
	InputParam("latents", required=True, type_hint=torch.Tensor),
	<span class="hljs-addition">+ InputParam("original_latents", type_hint=torch.Tensor),</span>
	<span class="hljs-addition">+ InputParam("diffdiff_masks", type_hint=torch.Tensor),</span>
	]

	def __call__(self, components, block_state, i, t):
	<span class="hljs-addition">+ # Apply differential diffusion logic</span>
	<span class="hljs-addition">+ if i == 0 and block_state.denoising_start is None:</span>
	<span class="hljs-addition">+ block_state.latents = block_state.original_latents[:1]</span>
	<span class="hljs-addition">+ else:</span>
	<span class="hljs-addition">+ block_state.mask = block_state.diffdiff_masks[i].unsqueeze(0).unsqueeze(1)</span>
	<span class="hljs-addition">+ block_state.latents = block_state.original_latents[i] * block_state.mask + block_state.latents * (1 - block_state.mask)</span>

	# ... rest of existing logic ...<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1c0d30o">That’s all there is to it! We’ve just created a simple sequential pipeline by mix-and-match some existing and new pipeline blocks.</p> <p data-svelte-h="svelte-o3bfrg">Now we use the process we’ve prepred in step2 to build the pipeline and inspect it.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->>> dd_pipeline
	SequentialPipelineBlocks(
	Class: ModularPipelineBlocks

	Description:


	Components:
	text_encoder (`CLIPTextModel`)
	text_encoder_2 (`CLIPTextModelWithProjection`)
	tokenizer (`CLIPTokenizer`)
	tokenizer_2 (`CLIPTokenizer`)
	guider (`ClassifierFreeGuidance`)
	vae (`AutoencoderKL`)
	image_processor (`VaeImageProcessor`)
	scheduler (`EulerDiscreteScheduler`)
	mask_processor (`VaeImageProcessor`)
	unet (`UNet2DConditionModel`)

	Configs:
	force_zeros_for_empty_prompt (default: <span class="hljs-literal">True</span>)
	requires_aesthetics_score (default: <span class="hljs-literal">False</span>)

	Blocks:
	[<span class="hljs-number">0</span>] text_encoder (StableDiffusionXLTextEncoderStep)
	Description: Text Encoder step that generate text_embeddings to guide the image generation

	[<span class="hljs-number">1</span>] image_encoder (StableDiffusionXLVaeEncoderStep)
	Description: Vae Encoder step that encode the <span class="hljs-built_in">input</span> image into a latent representation

	[<span class="hljs-number">2</span>] <span class="hljs-built_in">input</span> (StableDiffusionXLInputStep)
	Description: Input processing step that:
	<span class="hljs-number">1.</span> Determines `batch_size` <span class="hljs-keyword">and</span> `dtype` based on `prompt_embeds`
	<span class="hljs-number">2.</span> Adjusts <span class="hljs-built_in">input</span> tensor shapes based on `batch_size` (number of prompts) <span class="hljs-keyword">and</span> `num_images_per_prompt`

	All <span class="hljs-built_in">input</span> tensors are expected to have either batch_size=<span class="hljs-number">1</span> <span class="hljs-keyword">or</span> <span class="hljs-keyword">match</span> the batch_size
	of prompt_embeds. The tensors will be duplicated across the batch dimension to
	have a final batch_size of batch_size * num_images_per_prompt.

	[<span class="hljs-number">3</span>] set_timesteps (StableDiffusionXLSetTimestepsStep)
	Description: Step that sets the schedule<span class="hljs-string">r's timesteps for inference

	[4] prepare_latents (SDXLDiffDiffPrepareLatentsStep)
	Description: Step that prepares the latents for the differential diffusion generation process

	[5] prepare_add_cond (StableDiffusionXLImg2ImgPrepareAdditionalConditioningStep)
	Description: Step that prepares the additional conditioning for the image-to-image/inpainting generation process

	[6] denoise (SDXLDiffDiffDenoiseStep)
	Description: Pipeline block that iteratively denoise the latents over `timesteps`. The specific steps with each iteration can be customized with `sub_blocks` attributes

	[7] decode (StableDiffusionXLDecodeStep)
	Description: Step that decodes the denoised latents into images

	)</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-wdqdqg">Run the example now, you should see an apple with its right half transformed into a green pear.</p> <p data-svelte-h="svelte-5a63qd"><img src="https://cdn-uploads.huggingface.co/production/uploads/624ef9ba9d608e459387b34e/4zqJOz-35Q0i6jyUW3liL.png" alt="Image description"></p> <h2 class="relative group"><a id="adding-ip-adapter" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#adding-ip-adapter"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Adding IP-adapter</span></h2> <p data-svelte-h="svelte-16errsw">We provide an auto IP-adapter block that you can plug-and-play into your modular workflow. It’s an <code>AutoPipelineBlocks</code>, so it will only run when the user passes an IP adapter image. In this tutorial, we’ll focus on how to package it into your differential diffusion workflow. To learn more about <code>AutoPipelineBlocks</code>, see <a href="./auto_pipeline_blocks.md">here</a></p> <p data-svelte-h="svelte-dwbzmn">We talked about how to add IP-adapter into your workflow in the <a href="./modular_pipeline.md">Modular Pipeline Guide</a>. Let’s just go ahead to create the IP-adapter block.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> diffusers.modular_pipelines.stable_diffusion_xl.encoders <span class="hljs-keyword">import</span> StableDiffusionXLAutoIPAdapterStep
	<span class="hljs-meta">>>> </span>ip_adapter_block = StableDiffusionXLAutoIPAdapterStep()<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-c1au0n">We can directly add the ip-adapter block instance to the <code>diffdiff_blocks</code> that we created before. The <code>sub_blocks</code> attribute is a <code>InsertableDict</code>, so we’re able to insert the it at specific position (index <code>0</code> here).</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>dd_blocks.sub_blocks.insert(<span class="hljs-string">"ip_adapter"</span>, ip_adapter_block, <span class="hljs-number">0</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-ufmqyk">Take a look at the new diff-diff pipeline with ip-adapter!</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-built_in">print</span>(dd_blocks)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-hmexoo">The pipeline now lists ip-adapter as its first block, and tells you that it will run only if <code>ip_adapter_image</code> is provided. It also includes the two new components from ip-adpater: <code>image_encoder</code> and <code>feature_extractor</code></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->SequentialPipelineBlocks(
	<span class="hljs-title"> Class:</span> ModularPipelineBlocks

	====================================================================================================
	This pipeline contains blocks that are selected at runtime based on inputs.
	<span class="hljs-title"> Trigger Inputs:</span> {'ip_adapter_image'}
	Use `get_execution_blocks()` with input names to see selected blocks (e.g. `get_execution_blocks('ip_adapter_image')`).
	====================================================================================================


	<span class="hljs-title"> Description:</span>


	<span class="hljs-title"> Components:</span>
	image_encoder (`CLIPVisionModelWithProjection`)
	feature_extractor (`CLIPImageProcessor`)
	unet (`UNet2DConditionModel`)
	guider (`ClassifierFreeGuidance`)
	text_encoder (`CLIPTextModel`)
	text_encoder_2 (`CLIPTextModelWithProjection`)
	tokenizer (`CLIPTokenizer`)
	tokenizer_2 (`CLIPTokenizer`)
	vae (`AutoencoderKL`)
	image_processor (`VaeImageProcessor`)
	scheduler (`EulerDiscreteScheduler`)
	mask_processor (`VaeImageProcessor`)

	<span class="hljs-title"> Configs:</span>
	<span class="hljs-title"> force_zeros_for_empty_prompt (default:</span> <span class="hljs-literal">True</span>)
	<span class="hljs-title"> requires_aesthetics_score (default:</span> <span class="hljs-literal">False</span>)

	<span class="hljs-title"> Blocks:</span>
	[<span class="hljs-number">0</span>] ip_adapter (StableDiffusionXLAutoIPAdapterStep)
	<span class="hljs-title"> Description:</span> Run IP Adapter step if `ip_adapter_image` is provided.

	[<span class="hljs-number">1</span>] text_encoder (StableDiffusionXLTextEncoderStep)
	<span class="hljs-title"> Description:</span> Text Encoder step that generate text_embeddings to guide the image generation

	[<span class="hljs-number">2</span>] image_encoder (StableDiffusionXLVaeEncoderStep)
	<span class="hljs-title"> Description:</span> Vae Encoder step that encode the input image into a latent representation

	[<span class="hljs-number">3</span>] input (StableDiffusionXLInputStep)
	<span class="hljs-title"> Description: Input processing step that:</span>
	<span class="hljs-number">1</span>. Determines `batch_size` <span class="hljs-literal">and</span> `dtype` based on `prompt_embeds`
	<span class="hljs-number">2</span>. Adjusts input tensor shapes based on `batch_size` (number of prompts) <span class="hljs-literal">and</span> `num_images_per_prompt`

	All input tensors are expected to have either batch_size=<span class="hljs-number">1</span> <span class="hljs-literal">or</span> match the batch_size
	of prompt_embeds. The tensors will be duplicated across the batch dimension to
	have a final batch_size of batch_size * num_images_per_prompt.

	[<span class="hljs-number">4</span>] set_timesteps (StableDiffusionXLSetTimestepsStep)
	<span class="hljs-title"> Description:</span> Step that sets the scheduler's timesteps for inference

	[<span class="hljs-number">5</span>] prepare_latents (SDXLDiffDiffPrepareLatentsStep)
	<span class="hljs-title"> Description:</span> Step that prepares the latents for the differential diffusion generation process

	[<span class="hljs-number">6</span>] prepare_add_cond (StableDiffusionXLImg2ImgPrepareAdditionalConditioningStep)
	<span class="hljs-title"> Description:</span> Step that prepares the additional conditioning for the image-to-image/inpainting generation process

	[<span class="hljs-number">7</span>] denoise (SDXLDiffDiffDenoiseStep)
	<span class="hljs-title"> Description:</span> Pipeline block that iteratively denoise the latents over `timesteps`. The specific steps with each iteration can be customized with `sub_blocks` attributes

	[<span class="hljs-number">8</span>] decode (StableDiffusionXLDecodeStep)
	<span class="hljs-title"> Description:</span> Step that decodes the denoised latents into images

	)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-42nr41">Let’s test it out. We used an orange image to condition the generation via ip-addapter and we can see a slight orange color and texture in the final output.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>ip_adapter_block = StableDiffusionXLAutoIPAdapterStep()
	<span class="hljs-meta">>>> </span>dd_blocks.sub_blocks.insert(<span class="hljs-string">"ip_adapter"</span>, ip_adapter_block, <span class="hljs-number">0</span>)
	<span class="hljs-meta">>>> </span>
	<span class="hljs-meta">>>> </span>dd_pipeline = dd_blocks.init_pipeline(<span class="hljs-string">"YiYiXu/modular-demo-auto"</span>, collection=<span class="hljs-string">"diffdiff"</span>)
	<span class="hljs-meta">>>> </span>dd_pipeline.load_default_components(torch_dtype=torch.float16)
	<span class="hljs-meta">>>> </span>dd_pipeline.loader.load_ip_adapter(<span class="hljs-string">"h94/IP-Adapter"</span>, subfolder=<span class="hljs-string">"sdxl_models"</span>, weight_name=<span class="hljs-string">"ip-adapter_sdxl.bin"</span>)
	<span class="hljs-meta">>>> </span>dd_pipeline.loader.set_ip_adapter_scale(<span class="hljs-number">0.6</span>)
	<span class="hljs-meta">>>> </span>dd_pipeline = dd_pipeline.to(device)
	<span class="hljs-meta">>>> </span>
	<span class="hljs-meta">>>> </span>ip_adapter_image = load_image(<span class="hljs-string">"https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/diffdiff_orange.jpeg"</span>)
	<span class="hljs-meta">>>> </span>image = load_image(<span class="hljs-string">"https://huggingface.co/datasets/OzzyGT/testing-resources/resolve/main/differential/20240329211129_4024911930.png?download=true"</span>)
	<span class="hljs-meta">>>> </span>mask = load_image(<span class="hljs-string">"https://huggingface.co/datasets/OzzyGT/testing-resources/resolve/main/differential/gradient_mask.png?download=true"</span>)
	<span class="hljs-meta">>>> </span>
	<span class="hljs-meta">>>> </span>prompt = <span class="hljs-string">"a green pear"</span>
	<span class="hljs-meta">>>> </span>negative_prompt = <span class="hljs-string">"blurry"</span>
	<span class="hljs-meta">>>> </span>generator = torch.Generator(device=device).manual_seed(<span class="hljs-number">42</span>)
	<span class="hljs-meta">>>> </span>
	<span class="hljs-meta">>>> </span>image = dd_pipeline(
	<span class="hljs-meta">... </span> prompt=prompt,
	<span class="hljs-meta">... </span> negative_prompt=negative_prompt,
	<span class="hljs-meta">... </span> num_inference_steps=<span class="hljs-number">25</span>,
	<span class="hljs-meta">... </span> generator=generator,
	<span class="hljs-meta">... </span> ip_adapter_image=ip_adapter_image,
	<span class="hljs-meta">... </span> diffdiff_map=mask,
	<span class="hljs-meta">... </span> image=image,
	<span class="hljs-meta">... </span> output=<span class="hljs-string">"images"</span>
	<span class="hljs-meta">... </span>)[<span class="hljs-number">0</span>]<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="working-with-controlnets" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#working-with-controlnets"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Working with ControlNets</span></h2> <p data-svelte-h="svelte-ozranp">What about controlnet? Can differential diffusion work with controlnet? The key differences between a regular pipeline and a ControlNet pipeline are:</p> <ol data-svelte-h="svelte-1a4y58l"><li>A ControlNet input step that prepares the control condition</li> <li>Inside the denoising loop, a modified denoiser step where the control image is first processed through ControlNet, then control information is injected into the UNet</li></ol> <p data-svelte-h="svelte-cwx54i">From looking at the code workflow: differential diffusion only modifies the “before denoiser” step, while ControlNet operates within the “denoiser” itself. Since they intervene at different points in the pipeline, they should work together without conflicts.</p> <p data-svelte-h="svelte-15k9fsb">Intuitively, these two techniques are orthogonal and should combine naturally: differential diffusion controls how much the inference process can deviate from the original in each region, while ControlNet controls in what direction that change occurs.</p> <p data-svelte-h="svelte-1bqr5ww">With this understanding, let’s assemble the diffdiff-controlnet loop by combining the diffdiff before-denoiser step and controlnet denoiser step.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">class</span> <span class="hljs-title class_">SDXLDiffDiffControlNetDenoiseStep</span>(<span class="hljs-title class_ inherited__">StableDiffusionXLDenoiseLoopWrapper</span>):
	<span class="hljs-meta">... </span> block_classes = [SDXLDiffDiffLoopBeforeDenoiser, StableDiffusionXLControlNetLoopDenoiser, StableDiffusionXLDenoiseLoopAfterDenoiser]
	<span class="hljs-meta">... </span> block_names = [<span class="hljs-string">"before_denoiser"</span>, <span class="hljs-string">"denoiser"</span>, <span class="hljs-string">"after_denoiser"</span>]
	<span class="hljs-meta">>>> </span>
	<span class="hljs-meta">>>> </span>controlnet_denoise_block = SDXLDiffDiffControlNetDenoiseStep()
	<span class="hljs-meta">>>> </span><span class="hljs-comment"># print(controlnet_denoise)</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1jp82cg">We provide a auto controlnet input block that you can directly put into your workflow to proceess the <code>control_image</code>: similar to auto ip-adapter block, this step will only run if <code>control_image</code> input is passed from user. It work with both controlnet and controlnet union.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> diffusers.modular_pipelines.stable_diffusion_xl.modular_blocks <span class="hljs-keyword">import</span> StableDiffusionXLAutoControlNetInputStep
	<span class="hljs-meta">>>> </span>control_input_block = StableDiffusionXLAutoControlNetInputStep()
	<span class="hljs-meta">>>> </span><span class="hljs-built_in">print</span>(control_input_block)<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->StableDiffusionXLAutoControlNetInputStep(
	Class: AutoPipelineBlocks

	====================================================================================================
	This pipeline contains blocks that are selected at runtime based on inputs.
	Trigger Inputs: ['control_image', 'control_mode']
	====================================================================================================


	Description: Controlnet Input step that <span class="hljs-keyword">prepare</span> the controlnet input.
	This <span class="hljs-keyword">is</span> an auto pipeline block that works <span class="hljs-keyword">for</span> both controlnet <span class="hljs-keyword">and</span> controlnet_union.
	(it should be called <span class="hljs-keyword">right</span> before the denoise step) - <span class="hljs-symbol">`StableDiffusionXLControlNetUnionInputStep`</span> <span class="hljs-keyword">is</span> called <span class="hljs-keyword">to</span> <span class="hljs-keyword">prepare</span> the controlnet input <span class="hljs-keyword">when</span> <span class="hljs-symbol">`control_mode`</span> <span class="hljs-keyword">and</span> <span class="hljs-symbol">`control_image`</span> are provided.
	- <span class="hljs-symbol">`StableDiffusionXLControlNetInputStep`</span> <span class="hljs-keyword">is</span> called <span class="hljs-keyword">to</span> <span class="hljs-keyword">prepare</span> the controlnet input <span class="hljs-keyword">when</span> <span class="hljs-symbol">`control_image`</span> <span class="hljs-keyword">is</span> provided. - <span class="hljs-keyword">if</span> neither <span class="hljs-symbol">`control_mode`</span> nor <span class="hljs-symbol">`control_image`</span> <span class="hljs-keyword">is</span> provided, step will be skipped.


	Components:
	controlnet (<span class="hljs-symbol">`ControlNetUnionModel`</span>)
	control_image_processor (<span class="hljs-symbol">`VaeImageProcessor`</span>)

	Sub-Blocks:
	• controlnet_union [<span class="hljs-keyword">trigger</span>: control_mode] (StableDiffusionXLControlNetUnionInputStep)
	Description: step that prepares inputs <span class="hljs-keyword">for</span> the ControlNetUnion model

	• controlnet [<span class="hljs-keyword">trigger</span>: control_image] (StableDiffusionXLControlNetInputStep)
	Description: step that <span class="hljs-keyword">prepare</span> inputs <span class="hljs-keyword">for</span> controlnet

	)
	<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1fka27c">Let’s assemble the blocks and run an example using controlnet + differential diffusion. We used a tomato as <code>control_image</code>, so you can see that in the output, the right half that transformed into a pear had a tomato-like shape.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>dd_blocks.sub_blocks.insert(<span class="hljs-string">"controlnet_input"</span>, control_input_block, <span class="hljs-number">7</span>)
	<span class="hljs-meta">>>> </span>dd_blocks.sub_blocks[<span class="hljs-string">"denoise"</span>] = controlnet_denoise_block
	<span class="hljs-meta">>>> </span>
	<span class="hljs-meta">>>> </span>dd_pipeline = dd_blocks.init_pipeline(<span class="hljs-string">"YiYiXu/modular-demo-auto"</span>, collection=<span class="hljs-string">"diffdiff"</span>)
	<span class="hljs-meta">>>> </span>dd_pipeline.load_default_components(torch_dtype=torch.float16)
	<span class="hljs-meta">>>> </span>dd_pipeline = dd_pipeline.to(device)
	<span class="hljs-meta">>>> </span>
	<span class="hljs-meta">>>> </span>control_image = load_image(<span class="hljs-string">"https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/diffdiff_tomato_canny.jpeg"</span>)
	<span class="hljs-meta">>>> </span>image = load_image(<span class="hljs-string">"https://huggingface.co/datasets/OzzyGT/testing-resources/resolve/main/differential/20240329211129_4024911930.png?download=true"</span>)
	<span class="hljs-meta">>>> </span>mask = load_image(<span class="hljs-string">"https://huggingface.co/datasets/OzzyGT/testing-resources/resolve/main/differential/gradient_mask.png?download=true"</span>)
	<span class="hljs-meta">>>> </span>
	<span class="hljs-meta">>>> </span>prompt = <span class="hljs-string">"a green pear"</span>
	<span class="hljs-meta">>>> </span>negative_prompt = <span class="hljs-string">"blurry"</span>
	<span class="hljs-meta">>>> </span>generator = torch.Generator(device=device).manual_seed(<span class="hljs-number">42</span>)
	<span class="hljs-meta">>>> </span>
	<span class="hljs-meta">>>> </span>image = dd_pipeline(
	<span class="hljs-meta">... </span> prompt=prompt,
	<span class="hljs-meta">... </span> negative_prompt=negative_prompt,
	<span class="hljs-meta">... </span> num_inference_steps=<span class="hljs-number">25</span>,
	<span class="hljs-meta">... </span> generator=generator,
	<span class="hljs-meta">... </span> control_image=control_image,
	<span class="hljs-meta">... </span> controlnet_conditioning_scale=<span class="hljs-number">0.5</span>,
	<span class="hljs-meta">... </span> diffdiff_map=mask,
	<span class="hljs-meta">... </span> image=image,
	<span class="hljs-meta">... </span> output=<span class="hljs-string">"images"</span>
	<span class="hljs-meta">... </span>)[<span class="hljs-number">0</span>]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1xb1uxq">Optionally, We can combine <code>SDXLDiffDiffControlNetDenoiseStep</code> and <code>SDXLDiffDiffDenoiseStep</code> into a <code>AutoPipelineBlocks</code> so that same workflow can work with or without controlnet.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">class</span> <span class="hljs-title class_">SDXLDiffDiffAutoDenoiseStep</span>(<span class="hljs-title class_ inherited__">AutoPipelineBlocks</span>):
	<span class="hljs-meta">... </span> block_classes = [SDXLDiffDiffControlNetDenoiseStep, SDXLDiffDiffDenoiseStep]
	<span class="hljs-meta">... </span> block_names = [<span class="hljs-string">"controlnet_denoise"</span>, <span class="hljs-string">"denoise"</span>]
	<span class="hljs-meta">... </span> block_trigger_inputs = [<span class="hljs-string">"controlnet_cond"</span>, <span class="hljs-literal">None</span>]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-d9g24d"><code>SDXLDiffDiffAutoDenoiseStep</code> will run the ControlNet denoise step if <code>control_image</code> input is provided, otherwise it will run the regular denoise step.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-wy1vy">Note that it’s perfectly fine not to use <code>AutoPipelineBlocks</code>. In fact, we recommend only using <code>AutoPipelineBlocks</code> to package your workflow at the end once you’ve verified all your pipelines work as expected.</p></div> <p data-svelte-h="svelte-8tic47">Now you can create the differential diffusion preset that works with ip-adapter & controlnet.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>DIFFDIFF_AUTO_BLOCKS = IMAGE2IMAGE_BLOCKS.copy()
	<span class="hljs-meta">>>> </span>DIFFDIFF_AUTO_BLOCKS[<span class="hljs-string">"prepare_latents"</span>] = SDXLDiffDiffPrepareLatentsStep
	<span class="hljs-meta">>>> </span>DIFFDIFF_AUTO_BLOCKS[<span class="hljs-string">"set_timesteps"</span>] = TEXT2IMAGE_BLOCKS[<span class="hljs-string">"set_timesteps"</span>]
	<span class="hljs-meta">>>> </span>DIFFDIFF_AUTO_BLOCKS[<span class="hljs-string">"denoise"</span>] = SDXLDiffDiffAutoDenoiseStep
	<span class="hljs-meta">>>> </span>DIFFDIFF_AUTO_BLOCKS.insert(<span class="hljs-string">"ip_adapter"</span>, StableDiffusionXLAutoIPAdapterStep, <span class="hljs-number">0</span>)
	<span class="hljs-meta">>>> </span>DIFFDIFF_AUTO_BLOCKS.insert(<span class="hljs-string">"controlnet_input"</span>,StableDiffusionXLControlNetAutoInput, <span class="hljs-number">7</span>)
	<span class="hljs-meta">>>> </span>
	<span class="hljs-meta">>>> </span><span class="hljs-built_in">print</span>(DIFFDIFF_AUTO_BLOCKS)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-utdv0m">to use</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>dd_auto_blocks = SequentialPipelineBlocks.from_blocks_dict(DIFFDIFF_AUTO_BLOCKS)
	<span class="hljs-meta">>>> </span>dd_pipeline = dd_auto_blocks.init_pipeline(...)<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="creating-a-modular-repo" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#creating-a-modular-repo"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Creating a Modular Repo</span></h2> <p data-svelte-h="svelte-16uc9yx">You can easily share your differential diffusion workflow on the Hub by creating a modular repo. This is one created using the code we just wrote together: <a href="https://huggingface.co/YiYiXu/modular-diffdiff" rel="nofollow">https://huggingface.co/YiYiXu/modular-diffdiff</a></p> <p data-svelte-h="svelte-ag6wkn">To create a Modular Repo and share on hub, you just need to run <code>save_pretrained()</code> along with the <code>push_to_hub=True</code> flag. Note that if your pipeline contains custom block, you need to manually upload the code to the hub. But we are working on a command line tool to help you upload it very easily.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->dd_pipeline.save_pretrained(<span class="hljs-string">"YiYiXu/test_modular_doc"</span>, push_to_hub=<span class="hljs-literal">True</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1n7fos4">With a modular repo, it is very easy for the community to use the workflow you just created! Here is an example to use the differential-diffusion pipeline we just created and shared.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> diffusers.modular_pipelines <span class="hljs-keyword">import</span> ModularPipeline, ComponentsManager
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> torch
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image
	<span class="hljs-meta">>>> </span>
	<span class="hljs-meta">>>> </span>repo_id = <span class="hljs-string">"YiYiXu/modular-diffdiff-0704"</span>
	<span class="hljs-meta">>>> </span>
	<span class="hljs-meta">>>> </span>components = ComponentsManager()
	<span class="hljs-meta">>>> </span>
	<span class="hljs-meta">>>> </span>diffdiff_pipeline = ModularPipeline.from_pretrained(repo_id, trust_remote_code=<span class="hljs-literal">True</span>, components_manager=components, collection=<span class="hljs-string">"diffdiff"</span>)
	<span class="hljs-meta">>>> </span>diffdiff_pipeline.load_default_components(torch_dtype=torch.float16)
	<span class="hljs-meta">>>> </span>components.enable_auto_cpu_offload()<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-92hmpb">see more usage example on model card.</p> <h2 class="relative group"><a id="deploy-a-mellon-node" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#deploy-a-mellon-node"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>deploy a mellon node</span></h2> <p data-svelte-h="svelte-1i8wquf">[YIYI TODO: for now, here is an example of mellon node https://huggingface.co/YiYiXu/diff-diff-mellon]</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/modular_diffusers/end_to_end_guide.md" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span data-svelte-h="svelte-x0xyl0">></span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_o82a48 = {
	assets: "/docs/diffusers/pr_11686/en",
	base: "/docs/diffusers/pr_11686/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/diffusers/pr_11686/en/_app/immutable/entry/start.2b9667fb.js"),
	import("/docs/diffusers/pr_11686/en/_app/immutable/entry/app.a2a6117e.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 224],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 96.7 kB
Xet hash:: 57dca90b01ad850a9a7e605ac77a7b657d44ab14317c9ff6889b16cebae69f72

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.