Buckets:

hf-doc-build
/

doc

Files

xet

hf-doc-build/doc / diffusers /main /en /modular_diffusers /mellon.html

HuggingFaceDocBuilder

about 7 hours ago

download

raw

44.9 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Using Custom Blocks with Mellon","local":"using-custom-blocks-with-mellon","sections":[],"depth":2}">
	<link href="/docs/diffusers/main/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/entry/start.55138708.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/scheduler.53228c21.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/singletons.59fc0d32.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/index.e93d0901.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/paths.4f58d9a7.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/entry/app.9b1131ae.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/preload-helper.5c375679.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/index.100fac89.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/nodes/0.eac8817e.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/nodes/259.6f0c6220.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/CodeBlock.d30a6509.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/IconCopy.38cf8f56.js">
	<link rel="modulepreload" href="/docs/diffusers/main/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.0a9663c0.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Using Custom Blocks with Mellon","local":"using-custom-blocks-with-mellon","sections":[],"depth":2}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h2 class="relative group"><a id="using-custom-blocks-with-mellon" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#using-custom-blocks-with-mellon"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Using Custom Blocks with Mellon</span></h2> <p data-svelte-h="svelte-1hdq0fy"><a href="https://github.com/cubiq/Mellon" rel="nofollow">Mellon</a> is a visual workflow interface that integrates with Modular Diffusers and is designed for node-based workflows.</p> <blockquote class="warning" data-svelte-h="svelte-jx3j6s"><p>Mellon is in early development and not ready for production use yet. Consider this a sneak peek of how the integration works!</p></blockquote> <p data-svelte-h="svelte-y3gu57">Custom blocks work in Mellon out of the box - just need to add a <code>mellon_pipeline_config.json</code> to your repository. This config file tells Mellon how to render your block’s parameters as UI components.</p> <p data-svelte-h="svelte-m7xixg">Here’s what it looks like in action with the <a href="https://huggingface.co/diffusers/gemini-prompt-expander-mellon" rel="nofollow">Gemini Prompt Expander</a> block:</p> <p data-svelte-h="svelte-l2boam"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/modular_demo_dynamic.gif" alt="Mellon custom block demo"></p> <p data-svelte-h="svelte-1efs9dg">To use a modular diffusers custom block in Mellon:</p> <ol data-svelte-h="svelte-1kosvrs"><li>Drag a <strong>Dynamic Block Node</strong> from the ModularDiffusers section</li> <li>Enter the <code>repo_id</code> (e.g., <code>diffusers/gemini-prompt-expander-mellon</code>)</li> <li>Click <strong>Load Custom Block</strong></li> <li>The node transforms to show your block’s inputs and outputs</li></ol> <p data-svelte-h="svelte-lnv4i6">Now let’s walk through how to create this config for your own custom block.</p> <h2 class="relative group"><a id="steps-to-create-a-mellon-config" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#steps-to-create-a-mellon-config"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Steps to create a Mellon config</span></h2> <ol data-svelte-h="svelte-ndaccr"><li><strong>Specify Mellon types for your parameters</strong> - Each <code>InputParam</code>/<code>OutputParam</code> needs a type that tells Mellon what UI component to render (e.g., <code>"textbox"</code>, <code>"dropdown"</code>, <code>"image"</code>).</li> <li><strong>Generate <code>mellon_pipeline_config.json</code></strong> - Use our utility to generate a config template and push it to your Hub repository.</li> <li><strong>(Optional) Manually adjust the config</strong> - Fine-tune the generated config for your specific needs.</li></ol> <h2 class="relative group"><a id="specify-mellon-types-for-parameters" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#specify-mellon-types-for-parameters"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Specify Mellon types for parameters</span></h2> <p data-svelte-h="svelte-13s86u4">Mellon types determine how each parameter renders in the UI. If you don’t specify a type for a parameter, it will default to <code>"custom"</code>, which renders as a simple connection dot. You can always adjust this later in the generated config.</p> <table data-svelte-h="svelte-1uyrncp"><thead><tr><th>Type</th> <th>Input/Output</th> <th>Description</th></tr></thead> <tbody><tr><td><code>image</code></td> <td>Both</td> <td>Image (PIL Image)</td></tr> <tr><td><code>video</code></td> <td>Both</td> <td>Video</td></tr> <tr><td><code>text</code></td> <td>Both</td> <td>Text display</td></tr> <tr><td><code>textbox</code></td> <td>Input</td> <td>Text input</td></tr> <tr><td><code>dropdown</code></td> <td>Input</td> <td>Dropdown selection menu</td></tr> <tr><td><code>slider</code></td> <td>Input</td> <td>Slider for numeric values</td></tr> <tr><td><code>number</code></td> <td>Input</td> <td>Numeric input</td></tr> <tr><td><code>checkbox</code></td> <td>Input</td> <td>Boolean toggle</td></tr></tbody></table> <p data-svelte-h="svelte-j4t31t">For parameters that need more configuration (like dropdowns with options, or sliders with min/max values), pass a <code>MellonParam</code> instance directly instead of a string. You can use one of the class methods below, or create a fully custom one with <code>MellonParam(name, label, type, ...)</code>.</p> <table data-svelte-h="svelte-1aktlu5"><thead><tr><th>Method</th> <th>Description</th></tr></thead> <tbody><tr><td><code>MellonParam.Input.image(name)</code></td> <td>Image input</td></tr> <tr><td><code>MellonParam.Input.textbox(name, default)</code></td> <td>Text input as textarea</td></tr> <tr><td><code>MellonParam.Input.dropdown(name, options, default)</code></td> <td>Dropdown selection</td></tr> <tr><td><code>MellonParam.Input.slider(name, default, min, max, step)</code></td> <td>Slider for numeric values</td></tr> <tr><td><code>MellonParam.Input.number(name, default, min, max, step)</code></td> <td>Numeric input (no slider)</td></tr> <tr><td><code>MellonParam.Input.seed(name, default)</code></td> <td>Seed input with randomize button</td></tr> <tr><td><code>MellonParam.Input.checkbox(name, default)</code></td> <td>Boolean checkbox</td></tr> <tr><td><code>MellonParam.Input.model(name)</code></td> <td>Model input for diffusers components</td></tr> <tr><td><code>MellonParam.Output.image(name)</code></td> <td>Image output</td></tr> <tr><td><code>MellonParam.Output.video(name)</code></td> <td>Video output</td></tr> <tr><td><code>MellonParam.Output.text(name)</code></td> <td>Text output</td></tr> <tr><td><code>MellonParam.Output.model(name)</code></td> <td>Model output for diffusers components</td></tr></tbody></table> <p data-svelte-h="svelte-1tbv7k4">Choose one of the methods below to specify a Mellon type.</p> <h3 class="relative group"><a id="using-metadata-in-block-definitions" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#using-metadata-in-block-definitions"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Using metadata in block definitions</span></h3> <p data-svelte-h="svelte-xjvby6">If you’re defining a custom block from scratch, add <code>metadata={"mellon": "<type>"}</code> directly to your <code>InputParam</code> and <code>OutputParam</code> definitions. If you’re editing an existing custom block from the Hub, see <a href="./custom_blocks#editing-custom-blocks">Editing custom blocks</a> for how to download it locally.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">class</span> <span class="hljs-title class_">GeminiPromptExpander</span>(<span class="hljs-title class_ inherited__">ModularPipelineBlocks</span>):

	<span class="hljs-meta"> @property</span>
	<span class="hljs-keyword">def</span> <span class="hljs-title function_">inputs</span>(<span class="hljs-params">self</span>) -> <span class="hljs-type">List</span>[InputParam]:
	<span class="hljs-keyword">return</span> [
	InputParam(
	<span class="hljs-string">"prompt"</span>,
	type_hint=<span class="hljs-built_in">str</span>,
	required=<span class="hljs-literal">True</span>,
	description=<span class="hljs-string">"Prompt to use"</span>,
	metadata={<span class="hljs-string">"mellon"</span>: <span class="hljs-string">"textbox"</span>}, <span class="hljs-comment"># Text input</span>
	)
	]

	<span class="hljs-meta"> @property</span>
	<span class="hljs-keyword">def</span> <span class="hljs-title function_">intermediate_outputs</span>(<span class="hljs-params">self</span>) -> <span class="hljs-type">List</span>[OutputParam]:
	<span class="hljs-keyword">return</span> [
	OutputParam(
	<span class="hljs-string">"prompt"</span>,
	type_hint=<span class="hljs-built_in">str</span>,
	description=<span class="hljs-string">"Expanded prompt by the LLM"</span>,
	metadata={<span class="hljs-string">"mellon"</span>: <span class="hljs-string">"text"</span>}, <span class="hljs-comment"># Text output</span>
	),
	OutputParam(
	<span class="hljs-string">"old_prompt"</span>,
	type_hint=<span class="hljs-built_in">str</span>,
	description=<span class="hljs-string">"Old prompt provided by the user"</span>,
	<CopyLLMTxtMenu containerStyle=<span class="hljs-string">"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"</span>></CopyLLMTxtMenu>

	<span class="hljs-comment"># No metadata - we don't want to render this in UI</span>
	)
	]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-fr3ye1">For full control over UI configuration, pass a <code>MellonParam</code> instance directly:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> diffusers.modular_pipelines.mellon_node_utils <span class="hljs-keyword">import</span> MellonParam

	InputParam(
	<span class="hljs-string">"mode"</span>,
	type_hint=<span class="hljs-built_in">str</span>,
	default=<span class="hljs-string">"balanced"</span>,
	metadata={<span class="hljs-string">"mellon"</span>: MellonParam.Input.dropdown(<span class="hljs-string">"mode"</span>, options=[<span class="hljs-string">"fast"</span>, <span class="hljs-string">"balanced"</span>, <span class="hljs-string">"quality"</span>])},
	)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="using-inputtypes-and-outputtypes-when-generating-config" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#using-inputtypes-and-outputtypes-when-generating-config"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Using input_types and output_types when Generating Config</span></h3> <p data-svelte-h="svelte-t83lkr">If you’re working with an existing pipeline or prefer to keep your block definitions clean, specify types when generating the config using the <code>input_types/output_types</code> argument:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> diffusers.modular_pipelines.mellon_node_utils <span class="hljs-keyword">import</span> MellonPipelineConfig

	mellon_config = MellonPipelineConfig.from_custom_block(
	blocks,
	input_types={<span class="hljs-string">"prompt"</span>: <span class="hljs-string">"textbox"</span>},
	output_types={<span class="hljs-string">"prompt"</span>: <span class="hljs-string">"text"</span>}
	)<!-- HTML_TAG_END --></pre></div> <blockquote class="note" data-svelte-h="svelte-1jvk5hc"><p>When both <code>metadata</code> and <code>input_types</code>/<code>output_types</code> are specified, the arguments overrides <code>metadata</code>.</p></blockquote> <h2 class="relative group"><a id="generate-and-push-the-mellon-config" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#generate-and-push-the-mellon-config"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Generate and push the Mellon config</span></h2> <p data-svelte-h="svelte-2v0980">After adding metadata to your block, generate the default Mellon configuration template and push it to the Hub:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> ModularPipelineBlocks
	<span class="hljs-keyword">from</span> diffusers.modular_pipelines.mellon_node_utils <span class="hljs-keyword">import</span> MellonPipelineConfig

	<span class="hljs-comment"># load your custom blocks from your local dir</span>
	blocks = ModularPipelineBlocks.from_pretrained(<span class="hljs-string">"/path/local/folder"</span>, trust_remote_code=<span class="hljs-literal">True</span>)

	<span class="hljs-comment"># Generate the default config template</span>
	mellon_config = MellonPipelineConfig.from_custom_block(blocks)
	<span class="hljs-comment"># push the default template to `repo_id`, you will need to pass the same local folder path so that it will save the config locally first</span>
	mellon_config.save(
	local_dir=<span class="hljs-string">"/path/local/folder"</span>,
	repo_id= repo_id,
	push_to_hub=<span class="hljs-literal">True</span>
	)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1otnwu2">This creates a <code>mellon_pipeline_config.json</code> file in your repository.</p> <h2 class="relative group"><a id="review-and-adjust-the-config" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#review-and-adjust-the-config"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Review and adjust the config</span></h2> <p data-svelte-h="svelte-lukbqt">The generated template is a starting point - you may want to adjust it for your needs. Let’s walk through the generated config for the Gemini Prompt Expander:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-punctuation">{</span>
	<span class="hljs-attr">"label"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"Gemini Prompt Expander"</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"default_repo"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">""</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"default_dtype"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">""</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"node_params"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
	<span class="hljs-attr">"custom"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
	<span class="hljs-attr">"params"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
	<span class="hljs-attr">"prompt"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
	<span class="hljs-attr">"label"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"Prompt"</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"type"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"string"</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"display"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"textarea"</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"default"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">""</span>
	<span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"out_prompt"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
	<span class="hljs-attr">"label"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"Prompt"</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"type"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"string"</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"display"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"output"</span>
	<span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"old_prompt"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
	<span class="hljs-attr">"label"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"Old Prompt"</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"type"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"custom"</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"display"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"output"</span>
	<span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"doc"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
	<span class="hljs-attr">"label"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"Doc"</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"type"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"string"</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"display"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"output"</span>
	<span class="hljs-punctuation">}</span>
	<span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"input_names"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">[</span><span class="hljs-string">"prompt"</span><span class="hljs-punctuation">]</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"model_input_names"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">[</span><span class="hljs-punctuation">]</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"output_names"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">[</span><span class="hljs-string">"out_prompt"</span><span class="hljs-punctuation">,</span> <span class="hljs-string">"old_prompt"</span><span class="hljs-punctuation">,</span> <span class="hljs-string">"doc"</span><span class="hljs-punctuation">]</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"block_name"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"custom"</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"node_type"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"custom"</span>
	<span class="hljs-punctuation">}</span>
	<span class="hljs-punctuation">}</span>
	<span class="hljs-punctuation">}</span><!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="understanding-the-structure" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#understanding-the-structure"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Understanding the Structure</span></h3> <p data-svelte-h="svelte-1ig7crc">The <code>params</code> dict defines how each UI element renders. The <code>input_names</code>, <code>model_input_names</code>, and <code>output_names</code> lists map these UI elements to the underlying <a href="/docs/diffusers/main/en/api/modular_diffusers/pipeline_blocks#diffusers.ModularPipelineBlocks">ModularPipelineBlocks</a>’s I/O interface:</p> <table data-svelte-h="svelte-1k7kfjk"><thead><tr><th>Mellon Config</th> <th>ModularPipelineBlocks</th></tr></thead> <tbody><tr><td><code>input_names</code></td> <td><code>inputs</code> property</td></tr> <tr><td><code>model_input_names</code></td> <td><code>expected_components</code> property</td></tr> <tr><td><code>output_names</code></td> <td><code>intermediate_outputs</code> property</td></tr></tbody></table> <p data-svelte-h="svelte-vh5v8l">In this example: <code>prompt</code> is the only input. There are no model components, and outputs include <code>out_prompt</code>, <code>old_prompt</code>, and <code>doc</code>.</p> <p data-svelte-h="svelte-y5fwib">Now let’s look at the <code>params</code> dict:</p> <ul data-svelte-h="svelte-4pfel2"><li><p><strong><code>prompt</code></strong>: An input parameter with <code>display: "textarea"</code> (renders as a text input box), <code>label: "Prompt"</code> (shown in the UI), and <code>default: ""</code> (starts empty). The <code>type: "string"</code> field is important in Mellon because it determines which nodes can connect together - only matching types can be linked with “noodles”.</p></li> <li><p><strong><code>out_prompt</code></strong>: The expanded prompt output. The <code>out_</code> prefix was automatically added because the input and output share the same name (<code>prompt</code>), avoiding naming conflicts in the config. It has <code>display: "output"</code> which renders as an output socket.</p></li> <li><p><strong><code>old_prompt</code></strong>: Has <code>type: "custom"</code> because we didn’t specify metadata. This renders as a simple dot in the UI. Since we don’t actually want to expose this in the UI, we can remove it.</p></li> <li><p><strong><code>doc</code></strong>: The documentation output, automatically added to all custom blocks.</p></li></ul> <h3 class="relative group"><a id="making-adjustments" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#making-adjustments"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Making Adjustments</span></h3> <p data-svelte-h="svelte-1hm79va">Remove <code>old_prompt</code> from both <code>params</code> and <code>output_names</code> because you won’t need to use it.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-punctuation">{</span>
	<span class="hljs-attr">"label"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"Gemini Prompt Expander"</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"default_repo"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">""</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"default_dtype"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">""</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"node_params"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
	<span class="hljs-attr">"custom"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
	<span class="hljs-attr">"params"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
	<span class="hljs-attr">"prompt"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
	<span class="hljs-attr">"label"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"Prompt"</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"type"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"string"</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"display"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"textarea"</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"default"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">""</span>
	<span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"out_prompt"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
	<span class="hljs-attr">"label"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"Prompt"</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"type"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"string"</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"display"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"output"</span>
	<span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"doc"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
	<span class="hljs-attr">"label"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"Doc"</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"type"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"string"</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"display"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"output"</span>
	<span class="hljs-punctuation">}</span>
	<span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"input_names"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">[</span><span class="hljs-string">"prompt"</span><span class="hljs-punctuation">]</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"model_input_names"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">[</span><span class="hljs-punctuation">]</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"output_names"</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">[</span><span class="hljs-string">"out_prompt"</span><span class="hljs-punctuation">,</span> <span class="hljs-string">"doc"</span><span class="hljs-punctuation">]</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"block_name"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"custom"</span><span class="hljs-punctuation">,</span>
	<span class="hljs-attr">"node_type"</span><span class="hljs-punctuation">:</span> <span class="hljs-string">"custom"</span>
	<span class="hljs-punctuation">}</span>
	<span class="hljs-punctuation">}</span>
	<span class="hljs-punctuation">}</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-q5vhi0">See the final config at <a href="https://huggingface.co/diffusers/gemini-prompt-expander-mellon" rel="nofollow">diffusers/gemini-prompt-expander-mellon</a>.</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/modular_diffusers/mellon.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_19i7tg0 = {
	assets: "/docs/diffusers/main/en",
	base: "/docs/diffusers/main/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/diffusers/main/en/_app/immutable/entry/start.55138708.js"),
	import("/docs/diffusers/main/en/_app/immutable/entry/app.9b1131ae.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 259],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 44.9 kB
Xet hash:: 5453d3116b025b633ef4735a7dd1489ec7d685f2d137679f31d38a0138c62c59

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.