diffusers/pr_14545/en/_app/immutable/nodes/100.CkY5eYIv.js

import"../chunks/DsnmJJEf.js";import{i as y,h as w,C as L,H as o,D as t,E as O,s as N}from"../chunks/CmJXCtRL.js";import{p as j,o as B,s as e,f as C,a as T,b as E,c as d,d as v,r as a,n as s}from"../chunks/DK803DsY.js";const z='{"title":"StableAudio3DiTModel","local":"stableaudio3ditmodel","sections":[{"title":"StableAudio3DiTModel","local":"diffusers.StableAudio3DiTModel","sections":[],"depth":2},{"title":"StableAudio3DiTBlock","local":"diffusers.models.transformers.transformer_stable_audio3.StableAudio3DiTBlock","sections":[],"depth":2},{"title":"StableAudio3DiTModelOutput","local":"diffusers.models.transformers.transformer_stable_audio3.StableAudio3DiTModelOutput","sections":[],"depth":2}],"depth":1}';var G=v('<meta name="hf:doc:metadata"/>'),R=v(`<p></p> <!> <!> <p>A rectified-flow velocity-prediction Diffusion Transformer (DiT) for audio generation, used in <a href="https://stability.ai/news/stable-audio-3" rel="nofollow">Stable Audio 3</a>.</p> <p>Each <code>StableAudio3DiTBlock</code> performs:</p> <ol><li><strong>Self-attention</strong> — differential multi-head attention with rotary position embeddings (RoPE).</li> <li><strong>Cross-attention</strong> — attends to the token sequence from the T5Gemma text encoder.</li> <li><strong>Feed-forward</strong> — SwiGLU projection.</li></ol> <p>The model is conditioned on a <strong>timestep</strong> (exponential Fourier features → linear projection) and a <strong>global
conditioning vector</strong> (duration embedding from <a href="/docs/diffusers/pr_14545/en/api/pipelines/stable_audio_3#diffusers.StableAudio3DurationEmbedder">StableAudio3DurationEmbedder</a>).</p> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The Diffusion Transformer (DiT) backbone of <a href="https://stability.ai/news/stable-audio-3" rel="nofollow">Stable Audio 3</a>.</p> <p>The model takes a batch of noisy audio latents, a scalar timestep, a cross-attention context (projected text and
duration tokens), and a global duration embedding, and predicts the velocity field (rectified-flow objective).</p> <p>Conditioning:</p> <ul><li>Cross-attention context (<code>encoder_hidden_states</code>) is projected by <code>to_cond_embed</code>.</li> <li>The global duration embedding (<code>global_hidden_states</code>) is projected by <code>to_global_embed</code>, summed with the
timestep embedding, then expanded by <code>global_cond_embedder</code> into the per-block AdaLN modulation.</li> <li><code>local_add_cond</code> (inpainting) is projected per-block by <code>to_local_embed</code>.</li></ul> <p><code>num_memory_tokens</code> learnable tokens are prepended to the audio sequence inside the transformer and removed before
the output projection.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Single SA3 DiT transformer block.</p> <p>Order of operations:</p> <ol><li>AdaLN-modulated self-attention (partial RoPE, RMS QK-norm)</li> <li>Cross-attention to the text/duration context (plain RMS pre-norm)</li> <li>AdaLN-modulated SwiGLU feed-forward</li></ol> <p>The AdaLN modulation is <code>to_scale_shift_gate + global_modulation</code>, split into six chunks <code>(scale_attn, shift_attn, gate_attn, scale_ff, shift_ff, gate_ff)</code>. Each gated branch is scaled by <code>sigmoid(1 - gate)</code>. Cross-attention is <em>not</em> AdaLN-modulated, matching the reference (<code>cross_attend_norm</code> is a plain RMS norm).</p> <p>When <code>local_seq</code> is provided (inpainting), it is projected per-block by <code>to_local_embed</code> and added to the audio
positions of the residual stream after cross-attention (and before the feed-forward), matching the reference.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The output of <a href="/docs/diffusers/pr_14545/en/api/models/stable_audio_3_transformer#diffusers.StableAudio3DiTModel">StableAudio3DiTModel</a>.</p></div> <!> <p></p>`,1);function F(D,S){j(S,!1),B(()=>{new URLSearchParams(window.location.search).get("fw")}),y();var l=R();w("krmnxz",h=>{var g=G();N(g,"content",z),T(h,g)});var c=e(C(l),2);L(c,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var m=e(c,2);o(m,{title:"StableAudio3DiTModel",local:"stableaudio3ditmodel",headingTag:"h1"});var f=e(m,10);o(f,{title:"StableAudio3DiTModel",local:"diffusers.StableAudio3DiTModel",headingTag:"h2"});var i=e(f,2),u=d(i);t(u,{name:"class diffusers.StableAudio3DiTModel",anchor:"diffusers.StableAudio3DiTModel",source:"https://github.com/huggingface/diffusers/blob/vr_14545/src/diffusers/models/transformers/transformer_stable_audio3.py#L403",parameters:[{name:"io_channels",val:": int = 256"},{name:"patch_size",val:": int = 1"},{name:"embed_dim",val:": int = 1536"},{name:"depth",val:": int = 24"},{name:"num_heads",val:": int = 24"},{name:"cond_token_dim",val:": int = 768"},{name:"global_cond_dim",val:": int = 768"},{name:"local_add_cond_dim",val:": int = 257"},{name:"timestep_features_dim",val:": int = 256"},{name:"ff_mult",val:": float = 4.0"},{name:"num_memory_tokens",val:": int = 64"},{name:"use_differential_attention",val:": bool = True"}],parametersDescription:[{anchor:"diffusers.StableAudio3DiTModel.io_channels",description:"<strong>io_channels</strong> (<code>int</code>, defaults to 256) &#x2014; Number of latent channels.",name:"io_channels"},{anchor:"diffusers.StableAudio3DiTModel.patch_size",description:"<strong>patch_size</strong> (<code>int</code>, defaults to 1) &#x2014; Temporal patch size applied before the transformer.",name:"patch_size"},{anchor:"diffusers.StableAudio3DiTModel.embed_dim",description:"<strong>embed_dim</strong> (<code>int</code>, defaults to 1536) &#x2014; Transformer hidden dimension.",name:"embed_dim"},{anchor:"diffusers.StableAudio3DiTModel.depth",description:"<strong>depth</strong> (<code>int</code>, defaults to 24) &#x2014; Number of <code>StableAudio3DiTBlock</code> layers.",name:"depth"},{anchor:"diffusers.StableAudio3DiTModel.num_heads",description:"<strong>num_heads</strong> (<code>int</code>, defaults to 24) &#x2014; Number of attention heads.",name:"num_heads"},{anchor:"diffusers.StableAudio3DiTModel.cond_token_dim",description:"<strong>cond_token_dim</strong> (<code>int</code>, defaults to 768) &#x2014; Dimension of the cross-attention context tokens.",name:"cond_token_dim"},{anchor:"diffusers.StableAudio3DiTModel.global_cond_dim",description:"<strong>global_cond_dim</strong> (<code>int</code>, defaults to 768) &#x2014; Dimension of the global duration embedding.",name:"global_cond_dim"},{anchor:"diffusers.StableAudio3DiTModel.local_add_cond_dim",description:"<strong>local_add_cond_dim</strong> (<code>int</code>, defaults to 257) &#x2014; Channels of the local-additive (inpaint) tensor.",name:"local_add_cond_dim"},{anchor:"diffusers.StableAudio3DiTModel.timestep_features_dim",description:"<strong>timestep_features_dim</strong> (<code>int</code>, defaults to 256) &#x2014; Output dimension of the Fourier timestep features.",name:"timestep_features_dim"},{anchor:"diffusers.StableAudio3DiTModel.ff_mult",description:"<strong>ff_mult</strong> (<code>float</code>, defaults to 4.0) &#x2014; SwiGLU feed-forward expansion factor.",name:"ff_mult"},{anchor:"diffusers.StableAudio3DiTModel.num_memory_tokens",description:"<strong>num_memory_tokens</strong> (<code>int</code>, defaults to 64) &#x2014; Number of learnable memory tokens.",name:"num_memory_tokens"},{anchor:"diffusers.StableAudio3DiTModel.use_differential_attention",description:"<strong>use_differential_attention</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014; Enable differential self/cross attention.",name:"use_differential_attention"}]});var _=e(u,12),A=d(_);t(A,{name:"forward",anchor:"diffusers.StableAudio3DiTModel.forward",source:"https://github.com/huggingface/diffusers/blob/vr_14545/src/diffusers/models/transformers/transformer_stable_audio3.py#L521",parameters:[{name:"hidden_states",val:": Tensor"},{name:"timestep",val:": Tensor"},{name:"encoder_hidden_states",val:": Tensor"},{name:"global_hidden_states",val:": Tensor"},{name:"encoder_attention_mask",val:": typing.Optional[torch.Tensor] = None"},{name:"local_add_cond",val:": typing.Optional[torch.Tensor] = None"},{name:"return_dict",val:": bool = True"}],parametersDescription:[{anchor:"diffusers.StableAudio3DiTModel.forward.hidden_states",description:"<strong>hidden_states</strong> (<code>torch.Tensor</code>) &#x2014; Noisy latent audio <code>(batch, io_channels, T)</code>.",name:"hidden_states"},{anchor:"diffusers.StableAudio3DiTModel.forward.timestep",description:"<strong>timestep</strong> (<code>torch.Tensor</code>) &#x2014; Diffusion timestep <code>(batch,)</code> in <code>[0, 1]</code>.",name:"timestep"},{anchor:"diffusers.StableAudio3DiTModel.forward.encoder_hidden_states",description:"<strong>encoder_hidden_states</strong> (<code>torch.Tensor</code>) &#x2014; Cross-attention context <code>(batch, T_ctx, cond_token_dim)</code>.",name:"encoder_hidden_states"},{anchor:"diffusers.StableAudio3DiTModel.forward.global_hidden_states",description:"<strong>global_hidden_states</strong> (<code>torch.Tensor</code>) &#x2014; Global duration embedding <code>(batch, global_cond_dim)</code>.",name:"global_hidden_states"},{anchor:"diffusers.StableAudio3DiTModel.forward.encoder_attention_mask",description:"<strong>encoder_attention_mask</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014; Bool mask <code>(batch, T_ctx)</code>, <code>True</code> = valid.",name:"encoder_attention_mask"},{anchor:"diffusers.StableAudio3DiTModel.forward.local_add_cond",description:`<strong>local_add_cond</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014; Local-additive (inpaint) conditioning
<code>(batch, local_add_cond_dim, T)</code>.`,name:"local_add_cond"},{anchor:"diffusers.StableAudio3DiTModel.forward.return_dict",description:"<strong>return_dict</strong> (<code>bool</code>, defaults to <code>True</code>) &#x2014; Whether to return a <code>StableAudio3DiTModelOutput</code>.",name:"return_dict"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>the predicted velocity field, same shape as <code>hidden_states</code>.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>StableAudio3DiTModelOutput</code> or <code>tuple</code></p>
`}),a(_),a(i);var p=e(i,2);o(p,{title:"StableAudio3DiTBlock",local:"diffusers.models.transformers.transformer_stable_audio3.StableAudio3DiTBlock",headingTag:"h2"});var n=e(p,2),x=d(n);t(x,{name:"class diffusers.models.transformers.transformer_stable_audio3.StableAudio3DiTBlock",anchor:"diffusers.models.transformers.transformer_stable_audio3.StableAudio3DiTBlock",source:"https://github.com/huggingface/diffusers/blob/vr_14545/src/diffusers/models/transformers/transformer_stable_audio3.py#L316",parameters:[{name:"dim",val:": int"},{name:"context_dim",val:": int"},{name:"dim_heads",val:": int = 64"},{name:"use_differential",val:": bool = True"},{name:"ff_mult",val:": float = 4.0"},{name:"local_add_cond_dim",val:": int = 257"}]}),s(10),a(n);var b=e(n,2);o(b,{title:"StableAudio3DiTModelOutput",local:"diffusers.models.transformers.transformer_stable_audio3.StableAudio3DiTModelOutput",headingTag:"h2"});var r=e(b,2),M=d(r);t(M,{name:"class diffusers.models.transformers.transformer_stable_audio3.StableAudio3DiTModelOutput",anchor:"diffusers.models.transformers.transformer_stable_audio3.StableAudio3DiTModelOutput",source:"https://github.com/huggingface/diffusers/blob/vr_14545/src/diffusers/models/transformers/transformer_stable_audio3.py#L34",parameters:[{name:"sample",val:": Tensor"}],parametersDescription:[{anchor:"diffusers.models.transformers.transformer_stable_audio3.StableAudio3DiTModelOutput.sample",description:`<strong>sample</strong> (<code>torch.Tensor</code>) &#x2014;
The predicted velocity field, of the same shape as the input <code>hidden_states</code>.`,name:"sample"}]}),s(2),a(r);var k=e(r,2);O(k,{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/api/models/stable_audio_3_transformer.md"}),s(2),T(D,l),E()}export{F as component};