diffusers/pr_14287/en/_app/immutable/nodes/124.D__1PIj2.js

import"../chunks/DsnmJJEf.js";import{i as M,h as k,C as L,H as s,D as a,E as A,s as F}from"../chunks/BtE7mKSK.js";import{p as S,o as z,s as e,f as E,a as b,b as R,c as n,d as y,n as o,r as t}from"../chunks/jDjavuwI.js";const U='{"title":"Parallelism","local":"parallelism","sections":[{"title":"ParallelConfig","local":"diffusers.ParallelConfig","sections":[],"depth":2},{"title":"ContextParallelConfig","local":"diffusers.ContextParallelConfig","sections":[],"depth":2},{"title":"TensorParallelConfig","local":"diffusers.TensorParallelConfig","sections":[],"depth":2}],"depth":1}';var K=y('<meta name="hf:doc:metadata"/>'),V=y(`<p></p> <!> <!> <p>Parallelism strategies help speed up diffusion transformers by distributing computations across multiple devices, allowing for faster inference/training times. Refer to the <a href="../training/distributed_inference">Distributed inferece</a> guide to learn more.</p> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Configuration for applying different parallelisms.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Configuration for context parallelism.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Apply context parallel on a model.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Configuration for tensor parallelism.</p> <p>Tensor parallelism shards weight matrices (column-wise and row-wise) across devices. Each device computes a partial
result; an AllReduce/AllGather at layer boundaries reconstructs the full output. Uses <code>torch.distributed.tensor.parallelize_module</code> with <code>ColwiseParallel</code> / <code>RowwiseParallel</code> sharding styles. Supported
device types are <code>"cuda"</code> and <code>"neuron"</code>.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Apply tensor parallel on a model from its flat <code>_tp_plan</code>.</p></div> <!> <p></p>`,1);function B(C,x){S(x,!1),z(()=>{new URLSearchParams(window.location.search).get("fw")}),M();var m=V();k("1o3ncvv",_=>{var v=K();F(v,"content",U),b(_,v)});var g=e(E(m),2);L(g,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var f=e(g,2);s(f,{title:"Parallelism",local:"parallelism",headingTag:"h1"});var h=e(f,4);s(h,{title:"ParallelConfig",local:"diffusers.ParallelConfig",headingTag:"h2"});var r=e(h,2),P=n(r);a(P,{name:"class diffusers.ParallelConfig",anchor:"diffusers.ParallelConfig",source:"https://github.com/huggingface/diffusers/blob/vr_14287/src/diffusers/models/_modeling_parallel.py#L197",parameters:[{name:"context_parallel_config",val:": diffusers.models._modeling_parallel.ContextParallelConfig | None = None"},{name:"tensor_parallel_config",val:": diffusers.models._modeling_parallel.TensorParallelConfig | None = None"},{name:"_rank",val:": int = None"},{name:"_world_size",val:": int = None"},{name:"_device",val:": device = None"},{name:"_mesh",val:": DeviceMesh = None"}],parametersDescription:[{anchor:"diffusers.ParallelConfig.context_parallel_config",description:`<strong>context_parallel_config</strong> (<code>ContextParallelConfig</code>, <em>optional</em>) &#x2014;
Configuration for context parallelism.`,name:"context_parallel_config"},{anchor:"diffusers.ParallelConfig.tensor_parallel_config",description:`<strong>tensor_parallel_config</strong> (<code>TensorParallelConfig</code>, <em>optional</em>) &#x2014;
Configuration for tensor parallelism.`,name:"tensor_parallel_config"}]}),o(2),t(r);var p=e(r,2);s(p,{title:"ContextParallelConfig",local:"diffusers.ContextParallelConfig",headingTag:"h2"});var l=e(p,2),w=n(l);a(w,{name:"class diffusers.ContextParallelConfig",anchor:"diffusers.ContextParallelConfig",source:"https://github.com/huggingface/diffusers/blob/vr_14287/src/diffusers/models/_modeling_parallel.py#L41",parameters:[{name:"ring_degree",val:": int | None = None"},{name:"ulysses_degree",val:": int | None = None"},{name:"convert_to_fp32",val:": bool = True"},{name:"rotate_method",val:": typing.Literal['allgather', 'alltoall'] = 'allgather'"},{name:"mesh",val:": typing.Optional[torch.distributed.device_mesh.DeviceMesh] = None"},{name:"ulysses_anything",val:": bool = False"},{name:"ring_anything",val:": bool = False"},{name:"_rank",val:": int = None"},{name:"_world_size",val:": int = None"},{name:"_device",val:": device = None"},{name:"_mesh",val:": DeviceMesh = None"},{name:"_flattened_mesh",val:": DeviceMesh = None"},{name:"_ring_mesh",val:": DeviceMesh = None"},{name:"_ulysses_mesh",val:": DeviceMesh = None"},{name:"_ring_local_rank",val:": int = None"},{name:"_ulysses_local_rank",val:": int = None"}],parametersDescription:[{anchor:"diffusers.ContextParallelConfig.ring_degree",description:`<strong>ring_degree</strong> (<code>int</code>, <em>optional</em>, defaults to <code>1</code>) &#x2014;
Number of devices to use for Ring Attention. Sequence is split across devices. Each device computes
attention between its local Q and KV chunks passed sequentially around ring. Lower memory (only holds 1/N
of KV at a time), overlaps compute with communication, but requires N iterations to see all tokens. Best
for long sequences with limited memory/bandwidth. Number of devices to use for ring attention within a
context parallel region. Must be a divisor of the total number of devices in the context parallel mesh.`,name:"ring_degree"},{anchor:"diffusers.ContextParallelConfig.ulysses_degree",description:`<strong>ulysses_degree</strong> (<code>int</code>, <em>optional</em>, defaults to <code>1</code>) &#x2014;
Number of devices to use for Ulysses Attention. Sequence split is across devices. Each device computes
local QKV, then all-gathers all KV chunks to compute full attention in one pass. Higher memory (stores all
KV), requires high-bandwidth all-to-all communication, but lower latency. Best for moderate sequences with
good interconnect bandwidth.`,name:"ulysses_degree"},{anchor:"diffusers.ContextParallelConfig.convert_to_fp32",description:`<strong>convert_to_fp32</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to convert output and LSE to float32 for ring attention numerical stability.`,name:"convert_to_fp32"},{anchor:"diffusers.ContextParallelConfig.rotate_method",description:`<strong>rotate_method</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;allgather&quot;</code>) &#x2014;
Method to use for rotating key/value states across devices in ring attention. Currently, only <code>&quot;allgather&quot;</code>
is supported.`,name:"rotate_method"},{anchor:"diffusers.ContextParallelConfig.ulysses_anything",description:`<strong>ulysses_anything</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to enable &#x201C;Ulysses Anything&#x201D; mode, which supports arbitrary sequence lengths and head counts that
are not evenly divisible by <code>ulysses_degree</code>. When enabled, <code>ulysses_degree</code> must be greater than 1 and
<code>ring_degree</code> must be 1.`,name:"ulysses_anything"},{anchor:"diffusers.ContextParallelConfig.ring_anything",description:`<strong>ring_anything</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to enable &#x201C;Ring Anything&#x201D; mode, which supports arbitrary sequence lengths. When enabled,
<code>ring_degree</code> must be greater than 1 and <code>ulysses_degree</code> must be 1.`,name:"ring_anything"},{anchor:"diffusers.ContextParallelConfig.mesh",description:`<strong>mesh</strong> (<code>torch.distributed.device_mesh.DeviceMesh</code>, <em>optional</em>) &#x2014;
A custom device mesh to use for context parallelism. If provided, this mesh will be used instead of
creating a new one. This is useful when combining context parallelism with other parallelism strategies
(e.g., FSDP, tensor parallelism) that share the same device mesh. The mesh must have both &#x201C;ring&#x201D; and
&#x201C;ulysses&#x201D; dimensions. Use size 1 for dimensions not being used (e.g., <code>mesh_shape=(2, 1, 4)</code> with
<code>mesh_dim_names=(&quot;ring&quot;, &quot;ulysses&quot;, &quot;fsdp&quot;)</code> for ring attention only with FSDP).`,name:"mesh"}]}),o(2),t(l);var i=e(l,2),N=n(i);a(N,{name:"diffusers.hooks.apply_context_parallel",anchor:"diffusers.hooks.apply_context_parallel",source:"https://github.com/huggingface/diffusers/blob/vr_14287/src/diffusers/hooks/context_parallel.py#L80",parameters:[{name:"module",val:": Module"},{name:"parallel_config",val:": ContextParallelConfig"},{name:"plan",val:": dict"}]}),o(2),t(i);var u=e(i,2);s(u,{title:"TensorParallelConfig",local:"diffusers.TensorParallelConfig",headingTag:"h2"});var d=e(u,2),D=n(d);a(D,{name:"class diffusers.TensorParallelConfig",anchor:"diffusers.TensorParallelConfig",source:"https://github.com/huggingface/diffusers/blob/vr_14287/src/diffusers/models/_modeling_parallel.py#L157",parameters:[{name:"tp_degree",val:": int = 1"},{name:"mesh",val:": typing.Optional[torch.distributed.device_mesh.DeviceMesh] = None"},{name:"_rank",val:": int = None"},{name:"_world_size",val:": int = None"},{name:"_device",val:": device = None"},{name:"_mesh",val:": DeviceMesh = None"},{name:"_tp_degree",val:": int = None"}],parametersDescription:[{anchor:"diffusers.TensorParallelConfig.tp_degree",description:`<strong>tp_degree</strong> (<code>int</code>, defaults to <code>1</code>) &#x2014;
Number of devices to shard across. Must be a divisor of the number of attention heads (and FFN hidden
dimensions) of the model being parallelised.`,name:"tp_degree"},{anchor:"diffusers.TensorParallelConfig.mesh",description:`<strong>mesh</strong> (<code>torch.distributed.device_mesh.DeviceMesh</code>, <em>optional</em>) &#x2014;
A custom device mesh to use. If provided, <code>tp_degree</code> is inferred from <code>mesh.size()</code> and the argument is
ignored. Useful when combining TP with other parallelism strategies (e.g. CP) that share the same mesh.`,name:"mesh"}]}),o(4),t(d);var c=e(d,2),T=n(c);a(T,{name:"diffusers.hooks.apply_tensor_parallel",anchor:"diffusers.hooks.apply_tensor_parallel",source:"https://github.com/huggingface/diffusers/blob/vr_14287/src/diffusers/hooks/tensor_parallel.py#L243",parameters:[{name:"model",val:": Module"},{name:"config",val:": TensorParallelConfig"},{name:"tp_plan",val:": dict"}]}),o(2),t(c);var q=e(c,2);A(q,{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/api/parallel.md"}),o(2),b(C,m),R()}export{B as component};