Buckets:

hf-doc-build
/

doc-dev

hf-doc-build/doc-dev / diffusers /main /en /training /custom_diffusion.html

51.1 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Custom Diffusion","local":"custom-diffusion","sections":[{"title":"Script parameters","local":"script-parameters","sections":[{"title":"Prior preservation loss","local":"prior-preservation-loss","sections":[],"depth":3},{"title":"Regularization","local":"regularization","sections":[],"depth":3}],"depth":2},{"title":"Training script","local":"training-script","sections":[],"depth":2},{"title":"Launch the script","local":"launch-the-script","sections":[],"depth":2},{"title":"Next steps","local":"next-steps","sections":[],"depth":2}],"depth":1}">
	<link href="/docs/diffusers/main/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/entry/start.21e27d66.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/scheduler.8c3d61f6.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/singletons.1db06f6d.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/index.0997d446.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/paths.085e8bc8.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/entry/app.de4fb612.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/index.da70eac4.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/nodes/0.f6117ae5.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/nodes/184.ec4012b2.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/Tip.1d9b8c37.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/CodeBlock.00a903b3.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/EditOnGithub.1e64e623.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/HfOption.c1483eb1.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/stores.d6eecc38.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Custom Diffusion","local":"custom-diffusion","sections":[{"title":"Script parameters","local":"script-parameters","sections":[{"title":"Prior preservation loss","local":"prior-preservation-loss","sections":[],"depth":3},{"title":"Regularization","local":"regularization","sections":[],"depth":3}],"depth":2},{"title":"Training script","local":"training-script","sections":[],"depth":2},{"title":"Launch the script","local":"launch-the-script","sections":[],"depth":2},{"title":"Next steps","local":"next-steps","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="custom-diffusion" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#custom-diffusion"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Custom Diffusion</span></h1> <p data-svelte-h="svelte-zoobzt"><a href="https://huggingface.co/papers/2212.04488" rel="nofollow">Custom Diffusion</a> is a training technique for personalizing image generation models. Like Textual Inversion, DreamBooth, and LoRA, Custom Diffusion only requires a few (~4-5) example images. This technique works by only training weights in the cross-attention layers, and it uses a special word to represent the newly learned concept. Custom Diffusion is unique because it can also learn multiple concepts at the same time.</p> <p data-svelte-h="svelte-1pc1nnz">If you’re training on a GPU with limited vRAM, you should try enabling xFormers with <code>--enable_xformers_memory_efficient_attention</code> for faster training with lower vRAM requirements (16GB). To save even more memory, add <code>--set_grads_to_none</code> in the training argument to set the gradients to <code>None</code> instead of zero (this option can cause some issues, so if you experience any, try removing this parameter).</p> <p data-svelte-h="svelte-u5h0jz">This guide will explore the <a href="https://github.com/huggingface/diffusers/blob/main/examples/custom_diffusion/train_custom_diffusion.py" rel="nofollow">train_custom_diffusion.py</a> script to help you become more familiar with it, and how you can adapt it for your own use-case.</p> <p data-svelte-h="svelte-l7dm2q">Before running the script, make sure you install the library from source:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->git <span class="hljs-built_in">clone</span> https://github.com/huggingface/diffusers
	<span class="hljs-built_in">cd</span> diffusers
	pip install .<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1nnn36i">Navigate to the example folder with the training script and install the required dependencies:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-built_in">cd</span> examples/custom_diffusion
	pip install -r requirements.txt
	pip install clip-retrieval<!-- HTML_TAG_END --></pre></div> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-1qbiqsn">🤗 Accelerate is a library for helping you train on multiple GPUs/TPUs or with mixed-precision. It’ll automatically configure your training setup based on your hardware and environment. Take a look at the 🤗 Accelerate <a href="https://huggingface.co/docs/accelerate/quicktour" rel="nofollow">Quick tour</a> to learn more.</p></div> <p data-svelte-h="svelte-60q53m">Initialize an 🤗 Accelerate environment:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->accelerate config<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-di6juu">To setup a default 🤗 Accelerate environment without choosing any configurations:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->accelerate config default<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-tsz4qp">Or if your environment doesn’t support an interactive shell, like a notebook, you can use:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> write_basic_config

	write_basic_config()<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1fkfdql">Lastly, if you want to train a model on your own dataset, take a look at the <a href="create_dataset">Create a dataset for training</a> guide to learn how to create a dataset that works with the training script.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-lc7164">The following sections highlight parts of the training script that are important for understanding how to modify it, but it doesn’t cover every aspect of the script in detail. If you’re interested in learning more, feel free to read through the <a href="https://github.com/huggingface/diffusers/blob/main/examples/custom_diffusion/train_custom_diffusion.py" rel="nofollow">script</a> and let us know if you have any questions or concerns.</p></div> <h2 class="relative group"><a id="script-parameters" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#script-parameters"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Script parameters</span></h2> <p data-svelte-h="svelte-1t7lm99">The training script contains all the parameters to help you customize your training run. These are found in the <a href="https://github.com/huggingface/diffusers/blob/64603389da01082055a901f2883c4810d1144edb/examples/custom_diffusion/train_custom_diffusion.py#L319" rel="nofollow"><code>parse_args()</code></a> function. The function comes with default values, but you can also set your own values in the training command if you’d like.</p> <p data-svelte-h="svelte-or9ig">For example, to change the resolution of the input image:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->accelerate launch train_custom_diffusion.py \
	--resolution=256<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-j5tc3k">Many of the basic parameters are described in the <a href="dreambooth#script-parameters">DreamBooth</a> training guide, so this guide focuses on the parameters unique to Custom Diffusion:</p> <ul data-svelte-h="svelte-11ngzwo"><li><code>--freeze_model</code>: freezes the key and value parameters in the cross-attention layer; the default is <code>crossattn_kv</code>, but you can set it to <code>crossattn</code> to train all the parameters in the cross-attention layer</li> <li><code>--concepts_list</code>: to learn multiple concepts, provide a path to a JSON file containing the concepts</li> <li><code>--modifier_token</code>: a special word used to represent the learned concept</li> <li><code>--initializer_token</code>: a special word used to initialize the embeddings of the <code>modifier_token</code></li></ul> <h3 class="relative group"><a id="prior-preservation-loss" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#prior-preservation-loss"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Prior preservation loss</span></h3> <p data-svelte-h="svelte-1oc83h6">Prior preservation loss is a method that uses a model’s own generated samples to help it learn how to generate more diverse images. Because these generated sample images belong to the same class as the images you provided, they help the model retain what it has learned about the class and how it can use what it already knows about the class to make new compositions.</p> <p data-svelte-h="svelte-1mpzp5">Many of the parameters for prior preservation loss are described in the <a href="dreambooth#prior-preservation-loss">DreamBooth</a> training guide.</p> <h3 class="relative group"><a id="regularization" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#regularization"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Regularization</span></h3> <p data-svelte-h="svelte-13fgxpn">Custom Diffusion includes training the target images with a small set of real images to prevent overfitting. As you can imagine, this can be easy to do when you’re only training on a few images! Download 200 real images with <code>clip_retrieval</code>. The <code>class_prompt</code> should be the same category as the target images. These images are stored in <code>class_data_dir</code>.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->python retrieve.py --class_prompt <span class="hljs-built_in">cat</span> --class_data_dir real_reg/samples_cat --num_class_images 200<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-hwsu6x">To enable regularization, add the following parameters:</p> <ul data-svelte-h="svelte-6jy6u"><li><code>--with_prior_preservation</code>: whether to use prior preservation loss</li> <li><code>--prior_loss_weight</code>: controls the influence of the prior preservation loss on the model</li> <li><code>--real_prior</code>: whether to use a small set of real images to prevent overfitting</li></ul> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->accelerate launch train_custom_diffusion.py \
	--with_prior_preservation \
	--prior_loss_weight=1.0 \
	--class_data_dir=<span class="hljs-string">"./real_reg/samples_cat"</span> \
	--class_prompt=<span class="hljs-string">"cat"</span> \
	--real_prior=True \<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="training-script" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#training-script"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Training script</span></h2> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-17hnobu">A lot of the code in the Custom Diffusion training script is similar to the <a href="dreambooth#training-script">DreamBooth</a> script. This guide instead focuses on the code that is relevant to Custom Diffusion.</p></div> <p data-svelte-h="svelte-1owi8ao">The Custom Diffusion training script has two dataset classes:</p> <ul data-svelte-h="svelte-1fby5zy"><li><a href="https://github.com/huggingface/diffusers/blob/64603389da01082055a901f2883c4810d1144edb/examples/custom_diffusion/train_custom_diffusion.py#L165" rel="nofollow"><code>CustomDiffusionDataset</code></a>: preprocesses the images, class images, and prompts for training</li> <li><a href="https://github.com/huggingface/diffusers/blob/64603389da01082055a901f2883c4810d1144edb/examples/custom_diffusion/train_custom_diffusion.py#L148" rel="nofollow"><code>PromptDataset</code></a>: prepares the prompts for generating class images</li></ul> <p data-svelte-h="svelte-1jto000">Next, the <code>modifier_token</code> is <a href="https://github.com/huggingface/diffusers/blob/64603389da01082055a901f2883c4810d1144edb/examples/custom_diffusion/train_custom_diffusion.py#L811" rel="nofollow">added to the tokenizer</a>, converted to token ids, and the token embeddings are resized to account for the new <code>modifier_token</code>. Then the <code>modifier_token</code> embeddings are initialized with the embeddings of the <code>initializer_token</code>. All parameters in the text encoder are frozen, except for the token embeddings since this is what the model is trying to learn to associate with the concepts.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->params_to_freeze = itertools.chain(
	text_encoder.text_model.encoder.parameters(),
	text_encoder.text_model.final_layer_norm.parameters(),
	text_encoder.text_model.embeddings.position_embedding.parameters(),
	)
	freeze_params(params_to_freeze)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-12higtg">Now you’ll need to add the <a href="https://github.com/huggingface/diffusers/blob/64603389da01082055a901f2883c4810d1144edb/examples/custom_diffusion/train_custom_diffusion.py#L911C3-L911C3" rel="nofollow">Custom Diffusion weights</a> to the attention layers. This is a really important step for getting the shape and size of the attention weights correct, and for setting the appropriate number of attention processors in each UNet block.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->st = unet.state_dict()
	<span class="hljs-keyword">for</span> name, _ <span class="hljs-keyword">in</span> unet.attn_processors.items():
	cross_attention_dim = <span class="hljs-literal">None</span> <span class="hljs-keyword">if</span> name.endswith(<span class="hljs-string">"attn1.processor"</span>) <span class="hljs-keyword">else</span> unet.config.cross_attention_dim
	<span class="hljs-keyword">if</span> name.startswith(<span class="hljs-string">"mid_block"</span>):
	hidden_size = unet.config.block_out_channels[-<span class="hljs-number">1</span>]
	<span class="hljs-keyword">elif</span> name.startswith(<span class="hljs-string">"up_blocks"</span>):
	block_id = <span class="hljs-built_in">int</span>(name[<span class="hljs-built_in">len</span>(<span class="hljs-string">"up_blocks."</span>)])
	hidden_size = <span class="hljs-built_in">list</span>(<span class="hljs-built_in">reversed</span>(unet.config.block_out_channels))[block_id]
	<span class="hljs-keyword">elif</span> name.startswith(<span class="hljs-string">"down_blocks"</span>):
	block_id = <span class="hljs-built_in">int</span>(name[<span class="hljs-built_in">len</span>(<span class="hljs-string">"down_blocks."</span>)])
	hidden_size = unet.config.block_out_channels[block_id]
	layer_name = name.split(<span class="hljs-string">".processor"</span>)[<span class="hljs-number">0</span>]
	weights = {
	<span class="hljs-string">"to_k_custom_diffusion.weight"</span>: st[layer_name + <span class="hljs-string">".to_k.weight"</span>],
	<span class="hljs-string">"to_v_custom_diffusion.weight"</span>: st[layer_name + <span class="hljs-string">".to_v.weight"</span>],
	}
	<span class="hljs-keyword">if</span> train_q_out:
	weights[<span class="hljs-string">"to_q_custom_diffusion.weight"</span>] = st[layer_name + <span class="hljs-string">".to_q.weight"</span>]
	weights[<span class="hljs-string">"to_out_custom_diffusion.0.weight"</span>] = st[layer_name + <span class="hljs-string">".to_out.0.weight"</span>]
	weights[<span class="hljs-string">"to_out_custom_diffusion.0.bias"</span>] = st[layer_name + <span class="hljs-string">".to_out.0.bias"</span>]
	<span class="hljs-keyword">if</span> cross_attention_dim <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
	custom_diffusion_attn_procs[name] = attention_class(
	train_kv=train_kv,
	train_q_out=train_q_out,
	hidden_size=hidden_size,
	cross_attention_dim=cross_attention_dim,
	).to(unet.device)
	custom_diffusion_attn_procs[name].load_state_dict(weights)
	<span class="hljs-keyword">else</span>:
	custom_diffusion_attn_procs[name] = attention_class(
	train_kv=<span class="hljs-literal">False</span>,
	train_q_out=<span class="hljs-literal">False</span>,
	hidden_size=hidden_size,
	cross_attention_dim=cross_attention_dim,
	)
	<span class="hljs-keyword">del</span> st
	unet.set_attn_processor(custom_diffusion_attn_procs)
	custom_diffusion_layers = AttnProcsLayers(unet.attn_processors)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-bhf9v4">The <a href="https://github.com/huggingface/diffusers/blob/84cd9e8d01adb47f046b1ee449fc76a0c32dc4e2/examples/custom_diffusion/train_custom_diffusion.py#L982" rel="nofollow">optimizer</a> is initialized to update the cross-attention layer parameters:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->optimizer = optimizer_class(
	itertools.chain(text_encoder.get_input_embeddings().parameters(), custom_diffusion_layers.parameters())
	<span class="hljs-keyword">if</span> args.modifier_token <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>
	<span class="hljs-keyword">else</span> custom_diffusion_layers.parameters(),
	lr=args.learning_rate,
	betas=(args.adam_beta1, args.adam_beta2),
	weight_decay=args.adam_weight_decay,
	eps=args.adam_epsilon,
	)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-dnomct">In the <a href="https://github.com/huggingface/diffusers/blob/84cd9e8d01adb47f046b1ee449fc76a0c32dc4e2/examples/custom_diffusion/train_custom_diffusion.py#L1048" rel="nofollow">training loop</a>, it is important to only update the embeddings for the concept you’re trying to learn. This means setting the gradients of all the other token embeddings to zero:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">if</span> args.modifier_token <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
	<span class="hljs-keyword">if</span> accelerator.num_processes > <span class="hljs-number">1</span>:
	grads_text_encoder = text_encoder.module.get_input_embeddings().weight.grad
	<span class="hljs-keyword">else</span>:
	grads_text_encoder = text_encoder.get_input_embeddings().weight.grad
	index_grads_to_zero = torch.arange(<span class="hljs-built_in">len</span>(tokenizer)) != modifier_token_id[<span class="hljs-number">0</span>]
	<span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(modifier_token_id[<span class="hljs-number">1</span>:])):
	index_grads_to_zero = index_grads_to_zero & (
	torch.arange(<span class="hljs-built_in">len</span>(tokenizer)) != modifier_token_id[i]
	)
	grads_text_encoder.data[index_grads_to_zero, :] = grads_text_encoder.data[
	index_grads_to_zero, :
	].fill_(<span class="hljs-number">0</span>)<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="launch-the-script" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#launch-the-script"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Launch the script</span></h2> <p data-svelte-h="svelte-9dei1q">Once you’ve made all your changes or you’re okay with the default configuration, you’re ready to launch the training script! 🚀</p> <p data-svelte-h="svelte-mn6swm">In this guide, you’ll download and use these example <a href="https://www.cs.cmu.edu/~custom-diffusion/assets/data.zip" rel="nofollow">cat images</a>. You can also create and use your own dataset if you want (see the <a href="create_dataset">Create a dataset for training</a> guide).</p> <p data-svelte-h="svelte-1o5n4y1">Set the environment variable <code>MODEL_NAME</code> to a model id on the Hub or a path to a local model, <code>INSTANCE_DIR</code> to the path where you just downloaded the cat images to, and <code>OUTPUT_DIR</code> to where you want to save the model. You’ll use <code><new1></code> as the special word to tie the newly learned embeddings to. The script creates and saves model checkpoints and a pytorch_custom_diffusion_weights.bin file to your repository.</p> <p data-svelte-h="svelte-gvoz0n">To monitor training progress with Weights and Biases, add the <code>--report_to=wandb</code> parameter to the training command and specify a validation prompt with <code>--validation_prompt</code>. This is useful for debugging and saving intermediate results.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-1nab0gg">If you’re training on human faces, the Custom Diffusion team has found the following parameters to work well:</p> <ul data-svelte-h="svelte-1a7julv"><li><code>--learning_rate=5e-6</code></li> <li><code>--max_train_steps</code> can be anywhere between 1000 and 2000</li> <li><code>--freeze_model=crossattn</code></li> <li>use at least 15-20 images to train with</li></ul></div> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">single concept </div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">multiple concepts </div></div> <div class="language-select"><div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-built_in">export</span> MODEL_NAME=<span class="hljs-string">"CompVis/stable-diffusion-v1-4"</span>
	<span class="hljs-built_in">export</span> OUTPUT_DIR=<span class="hljs-string">"path-to-save-model"</span>
	<span class="hljs-built_in">export</span> INSTANCE_DIR=<span class="hljs-string">"./data/cat"</span>

	accelerate launch train_custom_diffusion.py \
	--pretrained_model_name_or_path=<span class="hljs-variable">$MODEL_NAME</span> \
	--instance_data_dir=<span class="hljs-variable">$INSTANCE_DIR</span> \
	--output_dir=<span class="hljs-variable">$OUTPUT_DIR</span> \
	--class_data_dir=./real_reg/samples_cat/ \
	--with_prior_preservation \
	--real_prior \
	--prior_loss_weight=1.0 \
	--class_prompt=<span class="hljs-string">"cat"</span> \
	--num_class_images=200 \
	--instance_prompt=<span class="hljs-string">"photo of a <new1> cat"</span> \
	--resolution=512 \
	--train_batch_size=2 \
	--learning_rate=1e-5 \
	--lr_warmup_steps=0 \
	--max_train_steps=250 \
	--scale_lr \
	--hflip \
	--modifier_token <span class="hljs-string">"<new1>"</span> \
	--validation_prompt=<span class="hljs-string">"<new1> cat sitting in a bucket"</span> \
	--report_to=<span class="hljs-string">"wandb"</span> \
	--push_to_hub<!-- HTML_TAG_END --></pre></div> </div> <p data-svelte-h="svelte-1mw1g2x">Once training is finished, you can use your new Custom Diffusion model for inference.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">single concept </div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">multiple concepts </div></div> <div class="language-select"><div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> torch
	<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline

	pipeline = DiffusionPipeline.from_pretrained(
	<span class="hljs-string">"CompVis/stable-diffusion-v1-4"</span>, torch_dtype=torch.float16,
	).to(<span class="hljs-string">"cuda"</span>)
	pipeline.unet.load_attn_procs(<span class="hljs-string">"path-to-save-model"</span>, weight_name=<span class="hljs-string">"pytorch_custom_diffusion_weights.bin"</span>)
	pipeline.load_textual_inversion(<span class="hljs-string">"path-to-save-model"</span>, weight_name=<span class="hljs-string">"<new1>.bin"</span>)

	image = pipeline(
	<span class="hljs-string">"<new1> cat sitting in a bucket"</span>,
	num_inference_steps=<span class="hljs-number">100</span>,
	guidance_scale=<span class="hljs-number">6.0</span>,
	eta=<span class="hljs-number">1.0</span>,
	).images[<span class="hljs-number">0</span>]
	image.save(<span class="hljs-string">"cat.png"</span>)<!-- HTML_TAG_END --></pre></div> </div> <h2 class="relative group"><a id="next-steps" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#next-steps"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Next steps</span></h2> <p data-svelte-h="svelte-nq8p2g">Congratulations on training a model with Custom Diffusion! 🎉 To learn more:</p> <ul data-svelte-h="svelte-1u8bbml"><li>Read the <a href="https://www.cs.cmu.edu/~custom-diffusion/" rel="nofollow">Multi-Concept Customization of Text-to-Image Diffusion</a> blog post to learn more details about the experimental results from the Custom Diffusion team.</li></ul> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/training/custom_diffusion.md" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span data-svelte-h="svelte-x0xyl0">></span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_1eu7tzz = {
	assets: "/docs/diffusers/main/en",
	base: "/docs/diffusers/main/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/diffusers/main/en/_app/immutable/entry/start.21e27d66.js"),
	import("/docs/diffusers/main/en/_app/immutable/entry/app.de4fb612.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 184],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 51.1 kB
Xet hash:: 0b26c28d172aeb61043da596577ff15d47e30d59c4070c28e0338f1d75d40e83

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.