Hugging Face's logo

Buckets:
hf-doc-build
/
doc-dev

Files
xet
hf-doc-build/doc-dev / diffusers /main /en /advanced_inference /outpaint.html
rtrm's picture
rtrm
download
raw
30.9 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Outpainting&quot;,&quot;local&quot;:&quot;outpainting&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Image preparation&quot;,&quot;local&quot;:&quot;image-preparation&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Outpaint&quot;,&quot;local&quot;:&quot;outpaint&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}">
 <link href="/docs/diffusers/main/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
 <link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/entry/start.21e27d66.js">
 <link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/scheduler.8c3d61f6.js">
 <link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/singletons.1db06f6d.js">
 <link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/index.0997d446.js">
 <link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/paths.085e8bc8.js">
 <link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/entry/app.de4fb612.js">
 <link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/index.da70eac4.js">
 <link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/nodes/0.f6117ae5.js">
 <link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/each.e59479a4.js">
 <link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/nodes/2.b92fb4e2.js">
 <link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/Tip.1d9b8c37.js">
 <link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/CodeBlock.00a903b3.js">
 <link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/EditOnGithub.1e64e623.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Outpainting&quot;,&quot;local&quot;:&quot;outpainting&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Image preparation&quot;,&quot;local&quot;:&quot;image-preparation&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Outpaint&quot;,&quot;local&quot;:&quot;outpaint&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="outpainting" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#outpainting"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Outpainting</span></h1> <p data-svelte-h="svelte-16ua5v0">Outpainting extends an image beyond its original boundaries, allowing you to add, replace, or modify visual elements in an image while preserving the original image. Like <a href="../using-diffusers/inpaint">inpainting</a>, you want to fill the white area (in this case, the area outside of the original image) with new visual elements while keeping the original image (represented by a mask of black pixels). There are a couple of ways to outpaint, such as with a <a href="https://hf.co/blog/OzzyGT/outpainting-controlnet" rel="nofollow">ControlNet</a> or with <a href="https://hf.co/blog/OzzyGT/outpainting-differential-diffusion" rel="nofollow">Differential Diffusion</a>.</p> <p data-svelte-h="svelte-6ts35n">This guide will show you how to outpaint with an inpainting model, ControlNet, and a ZoeDepth estimator.</p> <p data-svelte-h="svelte-ftdr0t">Before you begin, make sure you have the <a href="https://github.com/huggingface/controlnet_aux" rel="nofollow">controlnet_aux</a> library installed so you can use the ZoeDepth estimator.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->!pip install -q controlnet_aux<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="image-preparation" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#image-preparation"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Image preparation</span></h2> <p data-svelte-h="svelte-1qxxpsh">Start by picking an image to outpaint with and remove the background with a Space like <a href="https://hf.co/spaces/briaai/BRIA-RMBG-1.4" rel="nofollow">BRIA-RMBG-1.4</a>.</p> <iframe src="https://briaai-bria-rmbg-1-4.hf.space" frameborder="0" width="850" height="450"></iframe> <p data-svelte-h="svelte-1mizknk">For example, remove the background from this image of a pair of shoes.</p> <div class="flex flex-row gap-4" data-svelte-h="svelte-1jvhkj8"><div class="flex-1"><img class="rounded-xl" src="https://huggingface.co/datasets/stevhliu/testing-images/resolve/main/original-jordan.png"> <figcaption class="mt-2 text-center text-sm text-gray-500">original image</figcaption></div> <div class="flex-1"><img class="rounded-xl" src="https://huggingface.co/datasets/stevhliu/testing-images/resolve/main/no-background-jordan.png"> <figcaption class="mt-2 text-center text-sm text-gray-500">background removed</figcaption></div></div> <p data-svelte-h="svelte-1jea20f"><a href="../using-diffusers/sdxl">Stable Diffusion XL (SDXL)</a> models work best with 1024x1024 images, but you can resize the image to any size as long as your hardware has enough memory to support it. The transparent background in the image should also be replaced with a white background. Create a function (like the one below) that scales and pastes the image onto a white background.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> random
<span class="hljs-keyword">import</span> requests
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> controlnet_aux <span class="hljs-keyword">import</span> ZoeDetector
<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image, ImageOps
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> (
 AutoencoderKL,
 ControlNetModel,
 StableDiffusionXLControlNetPipeline,
 StableDiffusionXLInpaintPipeline,
)
<span class="hljs-keyword">def</span> <span class="hljs-title function_">scale_and_paste</span>(<span class="hljs-params">original_image</span>):
 aspect_ratio = original_image.width / original_image.height
<span class="hljs-keyword">if</span> original_image.width &gt; original_image.height:
 new_width = <span class="hljs-number">1024</span>
 new_height = <span class="hljs-built_in">round</span>(new_width / aspect_ratio)
 <span class="hljs-keyword">else</span>:
 new_height = <span class="hljs-number">1024</span>
 new_width = <span class="hljs-built_in">round</span>(new_height * aspect_ratio)
resized_original = original_image.resize((new_width, new_height), Image.LANCZOS)
 white_background = Image.new(<span class="hljs-string">&quot;RGBA&quot;</span>, (<span class="hljs-number">1024</span>, <span class="hljs-number">1024</span>), <span class="hljs-string">&quot;white&quot;</span>)
 x = (<span class="hljs-number">1024</span> - new_width) // <span class="hljs-number">2</span>
 y = (<span class="hljs-number">1024</span> - new_height) // <span class="hljs-number">2</span>
 white_background.paste(resized_original, (x, y), resized_original)
<span class="hljs-keyword">return</span> resized_original, white_background
original_image = Image.<span class="hljs-built_in">open</span>(
 requests.get(
 <span class="hljs-string">&quot;https://huggingface.co/datasets/stevhliu/testing-images/resolve/main/no-background-jordan.png&quot;</span>,
 stream=<span class="hljs-literal">True</span>,
 ).raw
).convert(<span class="hljs-string">&quot;RGBA&quot;</span>)
resized_img, white_bg_image = scale_and_paste(original_image)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-ksi0ug">To avoid adding unwanted extra details, use the ZoeDepth estimator to provide additional guidance during generation and to ensure the shoes remain consistent with the original image.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->zoe = ZoeDetector.from_pretrained(<span class="hljs-string">&quot;lllyasviel/Annotators&quot;</span>)
image_zoe = zoe(white_bg_image, detect_resolution=<span class="hljs-number">512</span>, image_resolution=<span class="hljs-number">1024</span>)
image_zoe<!-- HTML_TAG_END --></pre></div> <div class="flex justify-center" data-svelte-h="svelte-rp0vpr"><img src="https://huggingface.co/datasets/stevhliu/testing-images/resolve/main/zoedepth-jordan.png"></div> <h2 class="relative group"><a id="outpaint" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#outpaint"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Outpaint</span></h2> <p data-svelte-h="svelte-1p8ap7h">Once your image is ready, you can generate content in the white area around the shoes with <a href="https://hf.co/destitech/controlnet-inpaint-dreamer-sdxl" rel="nofollow">controlnet-inpaint-dreamer-sdxl</a>, a SDXL ControlNet trained for inpainting.</p> <p data-svelte-h="svelte-xaw95o">Load the inpainting ControlNet, ZoeDepth model, VAE and pass them to the <a href="/docs/diffusers/main/en/api/pipelines/controlnet_sdxl#diffusers.StableDiffusionXLControlNetPipeline">StableDiffusionXLControlNetPipeline</a>. Then you can create an optional <code>generate_image</code> function (for convenience) to outpaint an initial image.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->controlnets = [
 ControlNetModel.from_pretrained(
 <span class="hljs-string">&quot;destitech/controlnet-inpaint-dreamer-sdxl&quot;</span>, torch_dtype=torch.float16, variant=<span class="hljs-string">&quot;fp16&quot;</span>
 ),
 ControlNetModel.from_pretrained(
 <span class="hljs-string">&quot;diffusers/controlnet-zoe-depth-sdxl-1.0&quot;</span>, torch_dtype=torch.float16
 ),
]
vae = AutoencoderKL.from_pretrained(<span class="hljs-string">&quot;madebyollin/sdxl-vae-fp16-fix&quot;</span>, torch_dtype=torch.float16).to(<span class="hljs-string">&quot;cuda&quot;</span>)
pipeline = StableDiffusionXLControlNetPipeline.from_pretrained(
 <span class="hljs-string">&quot;SG161222/RealVisXL_V4.0&quot;</span>, torch_dtype=torch.float16, variant=<span class="hljs-string">&quot;fp16&quot;</span>, controlnet=controlnets, vae=vae
).to(<span class="hljs-string">&quot;cuda&quot;</span>)
<span class="hljs-keyword">def</span> <span class="hljs-title function_">generate_image</span>(<span class="hljs-params">prompt, negative_prompt, inpaint_image, zoe_image, seed: <span class="hljs-built_in">int</span> = <span class="hljs-literal">None</span></span>):
 <span class="hljs-keyword">if</span> seed <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
 seed = random.randint(<span class="hljs-number">0</span>, <span class="hljs-number">2</span>**<span class="hljs-number">32</span> - <span class="hljs-number">1</span>)
generator = torch.Generator(device=<span class="hljs-string">&quot;cpu&quot;</span>).manual_seed(seed)
image = pipeline(
 prompt,
 negative_prompt=negative_prompt,
 image=[inpaint_image, zoe_image],
 guidance_scale=<span class="hljs-number">6.5</span>,
 num_inference_steps=<span class="hljs-number">25</span>,
 generator=generator,
 controlnet_conditioning_scale=[<span class="hljs-number">0.5</span>, <span class="hljs-number">0.8</span>],
 control_guidance_end=[<span class="hljs-number">0.9</span>, <span class="hljs-number">0.6</span>],
 ).images[<span class="hljs-number">0</span>]
<span class="hljs-keyword">return</span> image
prompt = <span class="hljs-string">&quot;nike air jordans on a basketball court&quot;</span>
negative_prompt = <span class="hljs-string">&quot;&quot;</span>
temp_image = generate_image(prompt, negative_prompt, white_bg_image, image_zoe, <span class="hljs-number">908097</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-13kbyjh">Paste the original image over the initial outpainted image. You’ll improve the outpainted background in a later step.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->x = (<span class="hljs-number">1024</span> - resized_img.width) // <span class="hljs-number">2</span>
y = (<span class="hljs-number">1024</span> - resized_img.height) // <span class="hljs-number">2</span>
temp_image.paste(resized_img, (x, y), resized_img)
temp_image<!-- HTML_TAG_END --></pre></div> <div class="flex justify-center" data-svelte-h="svelte-1f58coy"><img src="https://huggingface.co/datasets/stevhliu/testing-images/resolve/main/initial-outpaint.png"></div> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-81ikpi">Now is a good time to free up some memory if you’re running low!</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->pipeline=<span class="hljs-literal">None</span>
torch.cuda.empty_cache()<!-- HTML_TAG_END --></pre></div></div> <p data-svelte-h="svelte-srw55e">Now that you have an initial outpainted image, load the <a href="/docs/diffusers/main/en/api/pipelines/stable_diffusion/stable_diffusion_xl#diffusers.StableDiffusionXLInpaintPipeline">StableDiffusionXLInpaintPipeline</a> with the <a href="https://hf.co/SG161222/RealVisXL_V4.0" rel="nofollow">RealVisXL</a> model to generate the final outpainted image with better quality.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->pipeline = StableDiffusionXLInpaintPipeline.from_pretrained(
 <span class="hljs-string">&quot;OzzyGT/RealVisXL_V4.0_inpainting&quot;</span>,
 torch_dtype=torch.float16,
 variant=<span class="hljs-string">&quot;fp16&quot;</span>,
 vae=vae,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-n4298l">Prepare a mask for the final outpainted image. To create a more natural transition between the original image and the outpainted background, blur the mask to help it blend better.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->mask = Image.new(<span class="hljs-string">&quot;L&quot;</span>, temp_image.size)
mask.paste(resized_img.split()[<span class="hljs-number">3</span>], (x, y))
mask = ImageOps.invert(mask)
final_mask = mask.point(<span class="hljs-keyword">lambda</span> p: p &gt; <span class="hljs-number">128</span> <span class="hljs-keyword">and</span> <span class="hljs-number">255</span>)
mask_blurred = pipeline.mask_processor.blur(final_mask, blur_factor=<span class="hljs-number">20</span>)
mask_blurred<!-- HTML_TAG_END --></pre></div> <div class="flex justify-center" data-svelte-h="svelte-1anr4fy"><img src="https://huggingface.co/datasets/stevhliu/testing-images/resolve/main/blurred-mask.png"></div> <p data-svelte-h="svelte-ktwg6k">Create a better prompt and pass it to the <code>generate_outpaint</code> function to generate the final outpainted image. Again, paste the original image over the final outpainted background.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">def</span> <span class="hljs-title function_">generate_outpaint</span>(<span class="hljs-params">prompt, negative_prompt, image, mask, seed: <span class="hljs-built_in">int</span> = <span class="hljs-literal">None</span></span>):
 <span class="hljs-keyword">if</span> seed <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
 seed = random.randint(<span class="hljs-number">0</span>, <span class="hljs-number">2</span>**<span class="hljs-number">32</span> - <span class="hljs-number">1</span>)
generator = torch.Generator(device=<span class="hljs-string">&quot;cpu&quot;</span>).manual_seed(seed)
image = pipeline(
 prompt,
 negative_prompt=negative_prompt,
 image=image,
 mask_image=mask,
 guidance_scale=<span class="hljs-number">10.0</span>,
 strength=<span class="hljs-number">0.8</span>,
 num_inference_steps=<span class="hljs-number">30</span>,
 generator=generator,
 ).images[<span class="hljs-number">0</span>]
<span class="hljs-keyword">return</span> image
prompt = <span class="hljs-string">&quot;high quality photo of nike air jordans on a basketball court, highly detailed&quot;</span>
negative_prompt = <span class="hljs-string">&quot;&quot;</span>
final_image = generate_outpaint(prompt, negative_prompt, temp_image, mask_blurred, <span class="hljs-number">7688778</span>)
x = (<span class="hljs-number">1024</span> - resized_img.width) // <span class="hljs-number">2</span>
y = (<span class="hljs-number">1024</span> - resized_img.height) // <span class="hljs-number">2</span>
final_image.paste(resized_img, (x, y), resized_img)
final_image<!-- HTML_TAG_END --></pre></div> <div class="flex justify-center" data-svelte-h="svelte-1r64xky"><img src="https://huggingface.co/datasets/stevhliu/testing-images/resolve/main/final-outpaint.png"></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/advanced_inference/outpaint.md" target="_blank"><span data-svelte-h="svelte-1kd6by1">&lt;</span> <span data-svelte-h="svelte-x0xyl0">&gt;</span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p>
<script>
 {
 __sveltekit_1eu7tzz = {
 assets: "/docs/diffusers/main/en",
 base: "/docs/diffusers/main/en",
 env: {}
 };
 const element = document.currentScript.parentElement;
 const data = [null,null];
 Promise.all([
 import("/docs/diffusers/main/en/_app/immutable/entry/start.21e27d66.js"),
 import("/docs/diffusers/main/en/_app/immutable/entry/app.de4fb612.js")
 ]).then(([kit, app]) => {
 kit.start(app, element, {
 node_ids: [0, 2],
 data,
 form: null,
 error: null
 });
 });
 }
 </script>

Xet Storage Details

Size:
30.9 kB
·
Xet hash:
842b212a4010cec76f9b387b5e89754f342e2930eafac414876c5ed53f016884

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.