Buckets:

hf-doc-build
/

doc

Files

xet

hf-doc-build/doc / diffusers /main /en /using-diffusers /custom_pipeline_examples.html

rtrm

17 days ago

download

raw

18.7 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Community pipelines","local":"community-pipelines","sections":[{"title":"Multilingual Stable Diffusion","local":"multilingual-stable-diffusion","sections":[],"depth":2},{"title":"MagicMix","local":"magicmix","sections":[],"depth":2}],"depth":1}">
	<link href="/docs/diffusers/main/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/entry/start.3a647a8f.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/scheduler.182ea377.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/singletons.d3235286.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/index.1f6d62f6.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/paths.c7219ec3.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/entry/app.819dd5bb.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/index.abf12888.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/nodes/0.6c016a30.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/nodes/170.86941089.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/Tip.230e2334.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/CodeBlock.57fe6e13.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/DocNotebookDropdown.5fa27ace.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/globals.7f7f1b26.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/EditOnGithub.9b8e78e4.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Community pipelines","local":"community-pipelines","sections":[{"title":"Multilingual Stable Diffusion","local":"multilingual-stable-diffusion","sections":[],"depth":2},{"title":"MagicMix","local":"magicmix","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="community-pipelines" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#community-pipelines"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Community pipelines</span></h1> <div class="flex space-x-1 absolute z-10 right-0 top-0"> <div class="relative colab-dropdown "> <button class=" " type="button"> <img alt="Open In Colab" class="!m-0" src="https://colab.research.google.com/assets/colab-badge.svg"> </button> </div> <div class="relative colab-dropdown "> <button class=" " type="button"> <img alt="Open In Studio Lab" class="!m-0" src="https://studiolab.sagemaker.aws/studiolab.svg"> </button> </div></div> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-18wmiaf">For more context about the design choices behind community pipelines, please have a look at <a href="https://github.com/huggingface/diffusers/issues/841" rel="nofollow">this issue</a>.</p></div> <p data-svelte-h="svelte-1baok02">Community pipelines allow you to get creative and build your own unique pipelines to share with the community. You can find all community pipelines in the <a href="https://github.com/huggingface/diffusers/tree/main/examples/community" rel="nofollow">diffusers/examples/community</a> folder along with inference and training examples for how to use them. This guide showcases some of the community pipelines and hopefully it’ll inspire you to create your own (feel free to open a PR with your own pipeline and we will merge it!).</p> <p data-svelte-h="svelte-1tl9wpb">To load a community pipeline, use the <code>custom_pipeline</code> argument in <a href="/docs/diffusers/main/en/api/pipelines/overview#diffusers.DiffusionPipeline">DiffusionPipeline</a> to specify one of the files in <a href="https://github.com/huggingface/diffusers/tree/main/examples/community" rel="nofollow">diffusers/examples/community</a>:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline

	pipe = DiffusionPipeline.from_pretrained(
	<span class="hljs-string">"CompVis/stable-diffusion-v1-4"</span>, custom_pipeline=<span class="hljs-string">"filename_in_the_community_folder"</span>, use_safetensors=<span class="hljs-literal">True</span>
	)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-19dq7yu">If a community pipeline doesn’t work as expected, please open a GitHub issue and mention the author.</p> <p data-svelte-h="svelte-ghaeyc">You can learn more about community pipelines in the how to <a href="custom_pipeline_overview">load community pipelines</a> and how to <a href="contribute_pipeline">contribute a community pipeline</a> guides.</p> <h2 class="relative group"><a id="multilingual-stable-diffusion" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#multilingual-stable-diffusion"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Multilingual Stable Diffusion</span></h2> <p data-svelte-h="svelte-1ee0giq">The multilingual Stable Diffusion pipeline uses a pretrained <a href="https://huggingface.co/papluca/xlm-roberta-base-language-detection" rel="nofollow">XLM-RoBERTa</a> to identify a language and the <a href="https://huggingface.co/facebook/mbart-large-50-many-to-one-mmt" rel="nofollow">mBART-large-50</a> model to handle the translation. This allows you to generate images from text in 20 languages.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> torch
	<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline
	<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> make_image_grid
	<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> (
	pipeline,
	MBart50TokenizerFast,
	MBartForConditionalGeneration,
	)

	device = <span class="hljs-string">"cuda"</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">"cpu"</span>
	device_dict = {<span class="hljs-string">"cuda"</span>: <span class="hljs-number">0</span>, <span class="hljs-string">"cpu"</span>: -<span class="hljs-number">1</span>}

	<span class="hljs-comment"># add language detection pipeline</span>
	language_detection_model_ckpt = <span class="hljs-string">"papluca/xlm-roberta-base-language-detection"</span>
	language_detection_pipeline = pipeline(<span class="hljs-string">"text-classification"</span>,
	model=language_detection_model_ckpt,
	device=device_dict[device])

	<span class="hljs-comment"># add model for language translation</span>
	translation_tokenizer = MBart50TokenizerFast.from_pretrained(<span class="hljs-string">"facebook/mbart-large-50-many-to-one-mmt"</span>)
	translation_model = MBartForConditionalGeneration.from_pretrained(<span class="hljs-string">"facebook/mbart-large-50-many-to-one-mmt"</span>).to(device)

	diffuser_pipeline = DiffusionPipeline.from_pretrained(
	<span class="hljs-string">"CompVis/stable-diffusion-v1-4"</span>,
	custom_pipeline=<span class="hljs-string">"multilingual_stable_diffusion"</span>,
	detection_pipeline=language_detection_pipeline,
	translation_model=translation_model,
	translation_tokenizer=translation_tokenizer,
	torch_dtype=torch.float16,
	)

	diffuser_pipeline.enable_attention_slicing()
	diffuser_pipeline = diffuser_pipeline.to(device)

	prompt = [<span class="hljs-string">"a photograph of an astronaut riding a horse"</span>,
	<span class="hljs-string">"Una casa en la playa"</span>,
	<span class="hljs-string">"Ein Hund, der Orange isst"</span>,
	<span class="hljs-string">"Un restaurant parisien"</span>]

	images = diffuser_pipeline(prompt).images
	make_image_grid(images, rows=<span class="hljs-number">2</span>, cols=<span class="hljs-number">2</span>)<!-- HTML_TAG_END --></pre></div> <div class="flex justify-center" data-svelte-h="svelte-1v1rdhr"><img src="https://user-images.githubusercontent.com/4313860/198328706-295824a4-9856-4ce5-8e66-278ceb42fd29.png"></div> <h2 class="relative group"><a id="magicmix" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#magicmix"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>MagicMix</span></h2> <p data-svelte-h="svelte-tjmn7p"><a href="https://huggingface.co/papers/2210.16056" rel="nofollow">MagicMix</a> is a pipeline that can mix an image and text prompt to generate a new image that preserves the image structure. The <code>mix_factor</code> determines how much influence the prompt has on the layout generation, <code>kmin</code> controls the number of steps during the content generation process, and <code>kmax</code> determines how much information is kept in the layout of the original image.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline, DDIMScheduler
	<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image, make_image_grid

	pipeline = DiffusionPipeline.from_pretrained(
	<span class="hljs-string">"CompVis/stable-diffusion-v1-4"</span>,
	custom_pipeline=<span class="hljs-string">"magic_mix"</span>,
	scheduler=DDIMScheduler.from_pretrained(<span class="hljs-string">"CompVis/stable-diffusion-v1-4"</span>, subfolder=<span class="hljs-string">"scheduler"</span>),
	).to(<span class="hljs-string">'cuda'</span>)

	img = load_image(<span class="hljs-string">"https://user-images.githubusercontent.com/59410571/209578593-141467c7-d831-4792-8b9a-b17dc5e47816.jpg"</span>)
	mix_img = pipeline(img, prompt=<span class="hljs-string">"bed"</span>, kmin=<span class="hljs-number">0.3</span>, kmax=<span class="hljs-number">0.5</span>, mix_factor=<span class="hljs-number">0.5</span>)
	make_image_grid([img, mix_img], rows=<span class="hljs-number">1</span>, cols=<span class="hljs-number">2</span>)<!-- HTML_TAG_END --></pre></div> <div class="flex gap-4" data-svelte-h="svelte-14yz3qw"><div><img class="rounded-xl" src="https://user-images.githubusercontent.com/59410571/209578593-141467c7-d831-4792-8b9a-b17dc5e47816.jpg"> <figcaption class="mt-2 text-center text-sm text-gray-500">original image</figcaption></div> <div><img class="rounded-xl" src="https://user-images.githubusercontent.com/59410571/209578602-70f323fa-05b7-4dd6-b055-e40683e37914.jpg"> <figcaption class="mt-2 text-center text-sm text-gray-500">image and text prompt mix</figcaption></div></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/using-diffusers/custom_pipeline_examples.md" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span data-svelte-h="svelte-x0xyl0">></span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_6uqa4y = {
	assets: "/docs/diffusers/main/en",
	base: "/docs/diffusers/main/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/diffusers/main/en/_app/immutable/entry/start.3a647a8f.js"),
	import("/docs/diffusers/main/en/_app/immutable/entry/app.819dd5bb.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 170],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 18.7 kB
Xet hash:: 5057482ad104427efc2c0745d97dd01100462406d0f9a0bba7c859b870becaa1

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.