Hugging Face's logo

Buckets:
hf-doc-build
/
doc-dev

Files
xet
hf-doc-build/doc-dev / diffusers /pr_11797 /en /quantization /overview.html
rtrm's picture
rtrm
download
raw
27.5 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Getting started&quot;,&quot;local&quot;:&quot;getting-started&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Pipeline-level quantization&quot;,&quot;local&quot;:&quot;pipeline-level-quantization&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Basic quantization&quot;,&quot;local&quot;:&quot;basic-quantization&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Advanced quantization&quot;,&quot;local&quot;:&quot;advanced-quantization&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Resources&quot;,&quot;local&quot;:&quot;resources&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}">
 <link href="/docs/diffusers/pr_11797/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
 <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/entry/start.2e1ee9a7.js">
 <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/chunks/scheduler.8c3d61f6.js">
 <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/chunks/singletons.702cc21f.js">
 <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/chunks/index.0997d446.js">
 <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/chunks/paths.ec7f49a7.js">
 <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/entry/app.87c95e21.js">
 <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/chunks/index.da70eac4.js">
 <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/nodes/0.7e207bc6.js">
 <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/chunks/each.e59479a4.js">
 <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/nodes/255.f56c91e8.js">
 <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/chunks/Tip.1d9b8c37.js">
 <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/chunks/CodeBlock.a9c4becf.js">
 <link rel="modulepreload" href="/docs/diffusers/pr_11797/en/_app/immutable/chunks/getInferenceSnippets.725ed3d4.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Getting started&quot;,&quot;local&quot;:&quot;getting-started&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Pipeline-level quantization&quot;,&quot;local&quot;:&quot;pipeline-level-quantization&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Basic quantization&quot;,&quot;local&quot;:&quot;basic-quantization&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Advanced quantization&quot;,&quot;local&quot;:&quot;advanced-quantization&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Resources&quot;,&quot;local&quot;:&quot;resources&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="getting-started" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#getting-started"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Getting started</span></h1> <p data-svelte-h="svelte-16bsuce">Quantization focuses on representing data with fewer bits while also trying to preserve the precision of the original data. This often means converting a data type to represent the same information with fewer bits. For example, if your model weights are stored as 32-bit floating points and they’re quantized to 16-bit floating points, this halves the model size which makes it easier to store and reduces memory usage. Lower precision can also speedup inference because it takes less time to perform calculations with fewer bits.</p> <p data-svelte-h="svelte-32yaio">Diffusers supports multiple quantization backends to make large diffusion models like <a href="../api/pipelines/flux">Flux</a> more accessible. This guide shows how to use the <a href="/docs/diffusers/pr_11797/en/api/quantization#diffusers.PipelineQuantizationConfig">PipelineQuantizationConfig</a> class to quantize a pipeline during its initialization from a pretrained or non-quantized checkpoint.</p> <h2 class="relative group"><a id="pipeline-level-quantization" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#pipeline-level-quantization"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Pipeline-level quantization</span></h2> <p data-svelte-h="svelte-s5db5z">There are two ways to use <a href="/docs/diffusers/pr_11797/en/api/quantization#diffusers.PipelineQuantizationConfig">PipelineQuantizationConfig</a> depending on how much customization you want to apply to the quantization configuration.</p> <ul data-svelte-h="svelte-1xtak7u"><li>for basic use cases, define the <code>quant_backend</code>, <code>quant_kwargs</code>, and <code>components_to_quantize</code> arguments</li> <li>for granular quantization control, define a <code>quant_mapping</code> that provides the quantization configuration for individual model components</li></ul> <h3 class="relative group"><a id="basic-quantization" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#basic-quantization"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Basic quantization</span></h3> <p data-svelte-h="svelte-1f16729">Initialize <a href="/docs/diffusers/pr_11797/en/api/quantization#diffusers.PipelineQuantizationConfig">PipelineQuantizationConfig</a> with the following parameters.</p> <ul data-svelte-h="svelte-10xkd3n"><li><code>quant_backend</code> specifies which quantization backend to use. Currently supported backends include: <code>bitsandbytes_4bit</code>, <code>bitsandbytes_8bit</code>, <code>gguf</code>, <code>quanto</code>, and <code>torchao</code>.</li> <li><code>quant_kwargs</code> specifies the quantization arguments to use.</li></ul> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-195iiez">These <code>quant_kwargs</code> arguments are different for each backend. Refer to the <a href="../api/quantization">Quantization API</a> docs to view the arguments for each backend.</p></div> <ul data-svelte-h="svelte-2kzewz"><li><code>components_to_quantize</code> specifies which components of the pipeline to quantize. Typically, you should quantize the most compute intensive components like the transformer. The text encoder is another component to consider quantizing if a pipeline has more than one such as <a href="/docs/diffusers/pr_11797/en/api/pipelines/flux#diffusers.FluxPipeline">FluxPipeline</a>. The example below quantizes the T5 text encoder in <a href="/docs/diffusers/pr_11797/en/api/pipelines/flux#diffusers.FluxPipeline">FluxPipeline</a> while keeping the CLIP model intact.</li></ul> <p data-svelte-h="svelte-3g0xtn">The example below loads the bitsandbytes backend with the following arguments from <a href="/docs/diffusers/pr_11797/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a>, <code>load_in_4bit</code>, <code>bnb_4bit_quant_type</code>, and <code>bnb_4bit_compute_dtype</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline
<span class="hljs-keyword">from</span> diffusers.quantizers <span class="hljs-keyword">import</span> PipelineQuantizationConfig
pipeline_quant_config = PipelineQuantizationConfig(
 quant_backend=<span class="hljs-string">&quot;bitsandbytes_4bit&quot;</span>,
 quant_kwargs={<span class="hljs-string">&quot;load_in_4bit&quot;</span>: <span class="hljs-literal">True</span>, <span class="hljs-string">&quot;bnb_4bit_quant_type&quot;</span>: <span class="hljs-string">&quot;nf4&quot;</span>, <span class="hljs-string">&quot;bnb_4bit_compute_dtype&quot;</span>: torch.bfloat16},
 components_to_quantize=[<span class="hljs-string">&quot;transformer&quot;</span>, <span class="hljs-string">&quot;text_encoder_2&quot;</span>],
)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1agf9et">Pass the <code>pipeline_quant_config</code> to <a href="/docs/diffusers/pr_11797/en/api/pipelines/overview#diffusers.DiffusionPipeline.from_pretrained">from_pretrained()</a> to quantize the pipeline.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->pipe = DiffusionPipeline.from_pretrained(
 <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
 quantization_config=pipeline_quant_config,
 torch_dtype=torch.bfloat16,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)
image = pipe(<span class="hljs-string">&quot;photo of a cute dog&quot;</span>).images[<span class="hljs-number">0</span>]<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="advanced-quantization" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#advanced-quantization"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Advanced quantization</span></h3> <p data-svelte-h="svelte-s8jyg6">The <code>quant_mapping</code> argument provides more options for how to quantize each individual component in a pipeline, like combining different quantization backends.</p> <p data-svelte-h="svelte-vx9g06">Initialize <a href="/docs/diffusers/pr_11797/en/api/quantization#diffusers.PipelineQuantizationConfig">PipelineQuantizationConfig</a> and pass a <code>quant_mapping</code> to it. The <code>quant_mapping</code> allows you to specify the quantization options for each component in the pipeline such as the transformer and text encoder.</p> <p data-svelte-h="svelte-s4tw4w">The example below uses two quantization backends, <a href="/docs/diffusers/pr_11797/en/api/quantization#diffusers.QuantoConfig">QuantoConfig</a> and <a href="https://huggingface.co/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig" rel="nofollow">transformers.BitsAndBytesConfig</a>, for the transformer and text encoder.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> DiffusersBitsAndBytesConfig
<span class="hljs-keyword">from</span> diffusers.quantizers.quantization_config <span class="hljs-keyword">import</span> QuantoConfig
<span class="hljs-keyword">from</span> diffusers.quantizers <span class="hljs-keyword">import</span> PipelineQuantizationConfig
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> TransformersBitsAndBytesConfig
pipeline_quant_config = PipelineQuantizationConfig(
 quant_mapping={
 <span class="hljs-string">&quot;transformer&quot;</span>: QuantoConfig(weights_dtype=<span class="hljs-string">&quot;int8&quot;</span>),
 <span class="hljs-string">&quot;text_encoder_2&quot;</span>: TransformersBitsAndBytesConfig(
 load_in_4bit=<span class="hljs-literal">True</span>, compute_dtype=torch.bfloat16
 ),
 }
)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-r6b6z6">There is a separate bitsandbytes backend in <a href="https://huggingface.co/docs/transformers/main_classes/quantization#transformers.BitsAndBytesConfig" rel="nofollow">Transformers</a>. You need to import and use <a href="https://huggingface.co/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig" rel="nofollow">transformers.BitsAndBytesConfig</a> for components that come from Transformers. For example, <code>text_encoder_2</code> in <a href="/docs/diffusers/pr_11797/en/api/pipelines/flux#diffusers.FluxPipeline">FluxPipeline</a> is a <a href="https://huggingface.co/docs/transformers/main/en/model_doc/t5#transformers.T5EncoderModel" rel="nofollow">T5EncoderModel</a> from Transformers so you need to use <a href="https://huggingface.co/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig" rel="nofollow">transformers.BitsAndBytesConfig</a> instead of <a href="/docs/diffusers/pr_11797/en/api/quantization#diffusers.BitsAndBytesConfig">diffusers.BitsAndBytesConfig</a>.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-1dohkit">Use the <a href="#basic-quantization">basic quantization</a> method above if you don’t want to manage these distinct imports or aren’t sure where each pipeline component comes from.</p></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> DiffusersBitsAndBytesConfig
<span class="hljs-keyword">from</span> diffusers.quantizers <span class="hljs-keyword">import</span> PipelineQuantizationConfig
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig <span class="hljs-keyword">as</span> TransformersBitsAndBytesConfig
pipeline_quant_config = PipelineQuantizationConfig(
 quant_mapping={
 <span class="hljs-string">&quot;transformer&quot;</span>: DiffusersBitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>, bnb_4bit_compute_dtype=torch.bfloat16),
 <span class="hljs-string">&quot;text_encoder_2&quot;</span>: TransformersBitsAndBytesConfig(
 load_in_4bit=<span class="hljs-literal">True</span>, compute_dtype=torch.bfloat16
 ),
 }
)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1agf9et">Pass the <code>pipeline_quant_config</code> to <a href="/docs/diffusers/pr_11797/en/api/pipelines/overview#diffusers.DiffusionPipeline.from_pretrained">from_pretrained()</a> to quantize the pipeline.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->pipe = DiffusionPipeline.from_pretrained(
 <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
 quantization_config=pipeline_quant_config,
 torch_dtype=torch.bfloat16,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)
image = pipe(<span class="hljs-string">&quot;photo of a cute dog&quot;</span>).images[<span class="hljs-number">0</span>]<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="resources" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#resources"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Resources</span></h2> <p data-svelte-h="svelte-kva6dz">Check out the resources below to learn more about quantization.</p> <ul data-svelte-h="svelte-yr6gzk"><li><p>If you are new to quantization, we recommend checking out the following beginner-friendly courses in collaboration with DeepLearning.AI.</p> <ul><li><a href="https://www.deeplearning.ai/short-courses/quantization-fundamentals-with-hugging-face/" rel="nofollow">Quantization Fundamentals with Hugging Face</a></li> <li><a href="https://www.deeplearning.ai/short-courses/quantization-in-depth/" rel="nofollow">Quantization in Depth</a></li></ul></li> <li><p>Refer to the <a href="https://huggingface.co/docs/transformers/main/en/quantization/contribute" rel="nofollow">Contribute new quantization method guide</a> if you’re interested in adding a new quantization method.</p></li> <li><p>The Transformers quantization <a href="https://huggingface.co/docs/transformers/quantization/overview#when-to-use-what" rel="nofollow">Overview</a> provides an overview of the pros and cons of different quantization backends.</p></li> <li><p>Read the <a href="https://huggingface.co/blog/diffusers-quantization" rel="nofollow">Exploring Quantization Backends in Diffusers</a> blog post for a brief introduction to each quantization backend, how to choose a backend, and combining quantization with other memory optimizations.</p></li></ul> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/quantization/overview.md" target="_blank"><span data-svelte-h="svelte-1kd6by1">&lt;</span> <span data-svelte-h="svelte-x0xyl0">&gt;</span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p>
<script>
 {
 __sveltekit_9fqn6m = {
 assets: "/docs/diffusers/pr_11797/en",
 base: "/docs/diffusers/pr_11797/en",
 env: {}
 };
 const element = document.currentScript.parentElement;
 const data = [null,null];
 Promise.all([
 import("/docs/diffusers/pr_11797/en/_app/immutable/entry/start.2e1ee9a7.js"),
 import("/docs/diffusers/pr_11797/en/_app/immutable/entry/app.87c95e21.js")
 ]).then(([kit, app]) => {
 kit.start(app, element, {
 node_ids: [0, 255],
 data,
 form: null,
 error: null
 });
 });
 }
 </script>

Xet Storage Details

Size:
27.5 kB
·
Xet hash:
a122eefdd39b72df792f349b764595449254118c22b6a598b13505b5f51d60e7

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.