Instructions to use diffusers-modular/krea2-edit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Diffusers
How to use diffusers-modular/krea2-edit with Diffusers:
pip install -U diffusers transformers accelerate
import torch from diffusers import DiffusionPipeline from diffusers.utils import load_image # switch to "mps" for apple devices pipe = DiffusionPipeline.from_pretrained("diffusers-modular/krea2-edit", torch_dtype=torch.bfloat16, device_map="cuda") prompt = "Turn this cat into a dog" input_image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png") image = pipe(image=input_image, prompt=prompt).images[0] - Notebooks
- Google Colab
- Kaggle
| # Copyright 2026 Krea AI and The HuggingFace Team. All rights reserved. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| import torch | |
| from diffusers.utils import logging | |
| from diffusers.modular_pipelines.modular_pipeline import AutoPipelineBlocks, SequentialPipelineBlocks | |
| from diffusers.modular_pipelines.modular_pipeline_utils import InputParam, InsertableDict, OutputParam | |
| from .before_denoise import ( | |
| Krea2CreateMaskLatentsStep, | |
| Krea2PrepareLatentsStep, | |
| Krea2PrepareLatentsWithStrengthStep, | |
| Krea2RoPEInputsStep, | |
| Krea2SetTimestepsStep, | |
| Krea2SetTimestepsWithStrengthStep, | |
| ) | |
| from .decoders import ( | |
| Krea2AfterDenoiseStep, | |
| Krea2DecoderStep, | |
| Krea2InpaintProcessImagesOutputStep, | |
| Krea2ProcessImagesOutputStep, | |
| ) | |
| from .denoise import ( | |
| Krea2DenoiseStep, | |
| Krea2InpaintDenoiseStep, | |
| ) | |
| from .encoders import ( | |
| Krea2InpaintProcessImagesInputStep, | |
| Krea2ProcessImagesInputStep, | |
| Krea2TextEncoderStep, | |
| Krea2VaeEncoderStep, | |
| ) | |
| from .inputs import ( | |
| Krea2AdditionalInputsStep, | |
| Krea2TextInputsStep, | |
| ) | |
| logger = logging.get_logger(__name__) | |
| # ==================== | |
| # 1. TEXT ENCODER | |
| # ==================== | |
| # auto_docstring | |
| class Krea2AutoTextEncoderStep(AutoPipelineBlocks): | |
| """ | |
| Text encoder step that encodes the text prompt into a text embedding. This is an auto pipeline block. | |
| - `Krea2TextEncoderStep` (text_encoder) is used when `prompt` is provided. | |
| - if `prompt` is not provided, step will be skipped. | |
| Components: | |
| text_encoder (`Qwen3VLModel`): The text encoder to use tokenizer (`Qwen2Tokenizer`): The tokenizer to use | |
| guider (`ClassifierFreeGuidance`) | |
| Inputs: | |
| prompt (`str`, *optional*): | |
| The prompt or prompts to guide image generation. | |
| negative_prompt (`str`, *optional*): | |
| The prompt or prompts not to guide the image generation. | |
| max_sequence_length (`int`, *optional*, defaults to 512): | |
| Maximum sequence length for prompt encoding. | |
| Outputs: | |
| prompt_embeds (`Tensor`): | |
| The prompt embeddings. | |
| prompt_embeds_mask (`Tensor`): | |
| The encoder attention mask. | |
| negative_prompt_embeds (`Tensor`): | |
| The negative prompt embeddings. | |
| negative_prompt_embeds_mask (`Tensor`): | |
| The negative prompt embeddings mask. | |
| """ | |
| model_name = "krea2" | |
| block_classes = [Krea2TextEncoderStep()] | |
| block_names = ["text_encoder"] | |
| block_trigger_inputs = ["prompt"] | |
| def description(self) -> str: | |
| return ( | |
| "Text encoder step that encodes the text prompt into a text embedding. This is an auto pipeline block.\n" | |
| " - `Krea2TextEncoderStep` (text_encoder) is used when `prompt` is provided.\n" | |
| " - if `prompt` is not provided, step will be skipped." | |
| ) | |
| # ==================== | |
| # 2. VAE ENCODER | |
| # ==================== | |
| # auto_docstring | |
| class Krea2InpaintVaeEncoderStep(SequentialPipelineBlocks): | |
| """ | |
| This step is used for processing image and mask inputs for inpainting tasks. It: | |
| - Resizes the image to the target size, based on `height` and `width`. | |
| - Processes and updates `image` and `mask_image`. | |
| - Creates `image_latents`. | |
| Components: | |
| image_mask_processor (`InpaintProcessor`) vae (`AutoencoderKLQwenImage`) | |
| Inputs: | |
| mask_image (`Image`): | |
| Mask image for inpainting. | |
| image (`Image | list`): | |
| Reference image(s) for denoising. Can be a single image or list of images. | |
| height (`int`, *optional*): | |
| The height in pixels of the generated image. | |
| width (`int`, *optional*): | |
| The width in pixels of the generated image. | |
| padding_mask_crop (`int`, *optional*): | |
| Padding for mask cropping in inpainting. | |
| generator (`Generator`, *optional*): | |
| Torch generator for deterministic generation. | |
| Outputs: | |
| processed_image (`Tensor`): | |
| The processed image | |
| processed_mask_image (`Tensor`): | |
| The processed mask image | |
| mask_overlay_kwargs (`dict`): | |
| The kwargs for the postprocess step to apply the mask overlay | |
| image_latents (`Tensor`): | |
| The latent representation of the input image. | |
| """ | |
| model_name = "krea2" | |
| block_classes = [Krea2InpaintProcessImagesInputStep(), Krea2VaeEncoderStep()] | |
| block_names = ["preprocess", "encode"] | |
| def description(self) -> str: | |
| return ( | |
| "This step is used for processing image and mask inputs for inpainting tasks. It:\n" | |
| " - Resizes the image to the target size, based on `height` and `width`.\n" | |
| " - Processes and updates `image` and `mask_image`.\n" | |
| " - Creates `image_latents`." | |
| ) | |
| # auto_docstring | |
| class Krea2Img2ImgVaeEncoderStep(SequentialPipelineBlocks): | |
| """ | |
| Vae encoder step that preprocess and encode the image inputs into their latent representations. | |
| Components: | |
| image_processor (`VaeImageProcessor`) vae (`AutoencoderKLQwenImage`) | |
| Inputs: | |
| image (`Image | list`): | |
| Reference image(s) for denoising. Can be a single image or list of images. | |
| height (`int`, *optional*): | |
| The height in pixels of the generated image. | |
| width (`int`, *optional*): | |
| The width in pixels of the generated image. | |
| generator (`Generator`, *optional*): | |
| Torch generator for deterministic generation. | |
| Outputs: | |
| processed_image (`Tensor`): | |
| The processed image | |
| image_latents (`Tensor`): | |
| The latent representation of the input image. | |
| """ | |
| model_name = "krea2" | |
| block_classes = [Krea2ProcessImagesInputStep(), Krea2VaeEncoderStep()] | |
| block_names = ["preprocess", "encode"] | |
| def description(self) -> str: | |
| return "Vae encoder step that preprocess and encode the image inputs into their latent representations." | |
| class Krea2AutoVaeEncoderStep(AutoPipelineBlocks): | |
| model_name = "krea2" | |
| block_classes = [Krea2InpaintVaeEncoderStep, Krea2Img2ImgVaeEncoderStep] | |
| block_names = ["inpaint", "img2img"] | |
| block_trigger_inputs = ["mask_image", "image"] | |
| def description(self): | |
| return ( | |
| "Vae encoder step that encode the image inputs into their latent representations.\n" | |
| + "This is an auto pipeline block.\n" | |
| + " - `Krea2InpaintVaeEncoderStep` (inpaint) is used when `mask_image` is provided.\n" | |
| + " - `Krea2Img2ImgVaeEncoderStep` (img2img) is used when `image` is provided.\n" | |
| + " - if `mask_image` or `image` is not provided, step will be skipped." | |
| ) | |
| # ==================== | |
| # 3. DENOISE (input -> prepare_latents -> set_timesteps -> prepare_rope_inputs -> denoise -> after_denoise) | |
| # ==================== | |
| # assemble input steps | |
| # auto_docstring | |
| class Krea2Img2ImgInputStep(SequentialPipelineBlocks): | |
| """ | |
| Input step that prepares the inputs for the img2img denoising step. It: | |
| - make sure the text embeddings have consistent batch size as well as the additional inputs (`image_latents`). | |
| - update height/width based `image_latents`, patchify `image_latents`. | |
| Components: | |
| pachifier (`Krea2Pachifier`) | |
| Inputs: | |
| num_images_per_prompt (`int`, *optional*, defaults to 1): | |
| The number of images to generate per prompt. | |
| prompt_embeds (`Tensor`): | |
| text embeddings used to guide the image generation. Can be generated from text_encoder step. | |
| prompt_embeds_mask (`Tensor`): | |
| mask for the text embeddings. Can be generated from text_encoder step. | |
| negative_prompt_embeds (`Tensor`, *optional*): | |
| negative text embeddings used to guide the image generation. Can be generated from text_encoder step. | |
| negative_prompt_embeds_mask (`Tensor`, *optional*): | |
| mask for the negative text embeddings. Can be generated from text_encoder step. | |
| height (`int`, *optional*): | |
| The height in pixels of the generated image. | |
| width (`int`, *optional*): | |
| The width in pixels of the generated image. | |
| image_latents (`Tensor`): | |
| image latents used to guide the image generation. Can be generated from vae_encoder step. | |
| Outputs: | |
| batch_size (`int`): | |
| The batch size of the prompt embeddings | |
| dtype (`dtype`): | |
| The data type of the prompt embeddings | |
| prompt_embeds (`Tensor`): | |
| The prompt embeddings. (batch-expanded) | |
| prompt_embeds_mask (`Tensor`): | |
| The encoder attention mask. (batch-expanded) | |
| negative_prompt_embeds (`Tensor`): | |
| The negative prompt embeddings. (batch-expanded) | |
| negative_prompt_embeds_mask (`Tensor`): | |
| The negative prompt embeddings mask. (batch-expanded) | |
| image_height (`int`): | |
| The image height calculated from the image latents dimension | |
| image_width (`int`): | |
| The image width calculated from the image latents dimension | |
| height (`int`): | |
| if not provided, updated to image height | |
| width (`int`): | |
| if not provided, updated to image width | |
| image_latents (`Tensor`): | |
| image latents used to guide the image generation. Can be generated from vae_encoder step. (patchified and | |
| batch-expanded) | |
| """ | |
| model_name = "krea2" | |
| block_classes = [Krea2TextInputsStep(), Krea2AdditionalInputsStep()] | |
| block_names = ["text_inputs", "additional_inputs"] | |
| def description(self): | |
| return ( | |
| "Input step that prepares the inputs for the img2img denoising step. It:\n" | |
| " - make sure the text embeddings have consistent batch size as well as the additional inputs (`image_latents`).\n" | |
| " - update height/width based `image_latents`, patchify `image_latents`." | |
| ) | |
| # auto_docstring | |
| class Krea2InpaintInputStep(SequentialPipelineBlocks): | |
| """ | |
| Input step that prepares the inputs for the inpainting denoising step. It: | |
| - make sure the text embeddings have consistent batch size as well as the additional inputs (`image_latents` and | |
| `processed_mask_image`). | |
| - update height/width based `image_latents`, patchify `image_latents`. | |
| Components: | |
| pachifier (`Krea2Pachifier`) | |
| Inputs: | |
| num_images_per_prompt (`int`, *optional*, defaults to 1): | |
| The number of images to generate per prompt. | |
| prompt_embeds (`Tensor`): | |
| text embeddings used to guide the image generation. Can be generated from text_encoder step. | |
| prompt_embeds_mask (`Tensor`): | |
| mask for the text embeddings. Can be generated from text_encoder step. | |
| negative_prompt_embeds (`Tensor`, *optional*): | |
| negative text embeddings used to guide the image generation. Can be generated from text_encoder step. | |
| negative_prompt_embeds_mask (`Tensor`, *optional*): | |
| mask for the negative text embeddings. Can be generated from text_encoder step. | |
| height (`int`, *optional*): | |
| The height in pixels of the generated image. | |
| width (`int`, *optional*): | |
| The width in pixels of the generated image. | |
| image_latents (`Tensor`, *optional*): | |
| image latents used to guide the image generation. Can be generated from vae_encoder step. | |
| processed_mask_image (`Tensor`, *optional*): | |
| The processed mask image | |
| Outputs: | |
| batch_size (`int`): | |
| The batch size of the prompt embeddings | |
| dtype (`dtype`): | |
| The data type of the prompt embeddings | |
| prompt_embeds (`Tensor`): | |
| The prompt embeddings. (batch-expanded) | |
| prompt_embeds_mask (`Tensor`): | |
| The encoder attention mask. (batch-expanded) | |
| negative_prompt_embeds (`Tensor`): | |
| The negative prompt embeddings. (batch-expanded) | |
| negative_prompt_embeds_mask (`Tensor`): | |
| The negative prompt embeddings mask. (batch-expanded) | |
| image_height (`int`): | |
| The image height calculated from the image latents dimension | |
| image_width (`int`): | |
| The image width calculated from the image latents dimension | |
| height (`int`): | |
| if not provided, updated to image height | |
| width (`int`): | |
| if not provided, updated to image width | |
| image_latents (`Tensor`): | |
| image latents used to guide the image generation. Can be generated from vae_encoder step. (patchified and | |
| batch-expanded) | |
| processed_mask_image (`Tensor`): | |
| The processed mask image (batch-expanded) | |
| """ | |
| model_name = "krea2" | |
| block_classes = [ | |
| Krea2TextInputsStep(), | |
| Krea2AdditionalInputsStep( | |
| additional_batch_inputs=[ | |
| InputParam(name="processed_mask_image", type_hint=torch.Tensor, description="The processed mask image") | |
| ] | |
| ), | |
| ] | |
| block_names = ["text_inputs", "additional_inputs"] | |
| def description(self): | |
| return ( | |
| "Input step that prepares the inputs for the inpainting denoising step. It:\n" | |
| " - make sure the text embeddings have consistent batch size as well as the additional inputs (`image_latents` and `processed_mask_image`).\n" | |
| " - update height/width based `image_latents`, patchify `image_latents`." | |
| ) | |
| # assemble prepare latents steps | |
| # auto_docstring | |
| class Krea2InpaintPrepareLatentsStep(SequentialPipelineBlocks): | |
| """ | |
| This step prepares the latents/image_latents and mask inputs for the inpainting denoising step. It: | |
| - Add noise to the image latents to create the latents input for the denoiser. | |
| - Create the patchified latents `mask` based on the processed mask image. | |
| Components: | |
| scheduler (`FlowMatchEulerDiscreteScheduler`) pachifier (`Krea2Pachifier`) | |
| Inputs: | |
| latents (`Tensor`): | |
| The initial random noised, can be generated in prepare latent step. | |
| image_latents (`Tensor`): | |
| image latents used to guide the image generation. Can be generated from vae_encoder step. (Can be | |
| generated from vae encoder and updated in input step.) | |
| timesteps (`Tensor`): | |
| The timesteps to use for the denoising process. Can be generated in set_timesteps step. | |
| processed_mask_image (`Tensor`): | |
| The processed mask to use for the inpainting process. | |
| height (`int`): | |
| The height in pixels of the generated image. | |
| width (`int`): | |
| The width in pixels of the generated image. | |
| dtype (`dtype`, *optional*, defaults to torch.float32): | |
| The dtype of the model inputs, can be generated in input step. | |
| Outputs: | |
| initial_noise (`Tensor`): | |
| The initial random noised used for inpainting denoising. | |
| latents (`Tensor`): | |
| The scaled noisy latents to use for inpainting/image-to-image denoising. | |
| mask (`Tensor`): | |
| The mask to use for the inpainting process. | |
| """ | |
| model_name = "krea2" | |
| block_classes = [Krea2PrepareLatentsWithStrengthStep(), Krea2CreateMaskLatentsStep()] | |
| block_names = ["add_noise_to_latents", "create_mask_latents"] | |
| def description(self) -> str: | |
| return ( | |
| "This step prepares the latents/image_latents and mask inputs for the inpainting denoising step. It:\n" | |
| " - Add noise to the image latents to create the latents input for the denoiser.\n" | |
| " - Create the patchified latents `mask` based on the processed mask image.\n" | |
| ) | |
| # assemble denoising steps | |
| # Krea 2 (text2image) | |
| # auto_docstring | |
| class Krea2CoreDenoiseStep(SequentialPipelineBlocks): | |
| """ | |
| step that denoise noise into image for text2image task. It includes the denoise loop, as well as prepare the inputs | |
| (timesteps, latents, rope inputs etc.). | |
| Components: | |
| pachifier (`Krea2Pachifier`) scheduler (`FlowMatchEulerDiscreteScheduler`) guider (`ClassifierFreeGuidance`) | |
| transformer (`Krea2Transformer2DModel`) | |
| Inputs: | |
| num_images_per_prompt (`int`, *optional*, defaults to 1): | |
| The number of images to generate per prompt. | |
| prompt_embeds (`Tensor`): | |
| text embeddings used to guide the image generation. Can be generated from text_encoder step. | |
| prompt_embeds_mask (`Tensor`): | |
| mask for the text embeddings. Can be generated from text_encoder step. | |
| negative_prompt_embeds (`Tensor`, *optional*): | |
| negative text embeddings used to guide the image generation. Can be generated from text_encoder step. | |
| negative_prompt_embeds_mask (`Tensor`, *optional*): | |
| mask for the negative text embeddings. Can be generated from text_encoder step. | |
| latents (`Tensor`, *optional*): | |
| Pre-generated noisy latents for image generation. | |
| height (`int`, *optional*): | |
| The height in pixels of the generated image. | |
| width (`int`, *optional*): | |
| The width in pixels of the generated image. | |
| generator (`Generator`, *optional*): | |
| Torch generator for deterministic generation. | |
| num_inference_steps (`int`, *optional*, defaults to 28): | |
| The number of denoising steps. | |
| sigmas (`list`, *optional*): | |
| Custom sigmas for the denoising process. | |
| mu (`float`, *optional*): | |
| Fixed timestep shift for the scheduler. Pass `1.15` for the few-step distilled (TDM/turbo) checkpoint; if | |
| not provided, computed from the image sequence length (base checkpoint behavior). | |
| **denoiser_input_fields (`None`, *optional*): | |
| conditional model inputs for the denoiser: e.g. prompt_embeds, negative_prompt_embeds, etc. | |
| Outputs: | |
| latents (`Tensor`): | |
| Denoised latents. | |
| """ | |
| model_name = "krea2" | |
| block_classes = [ | |
| Krea2TextInputsStep(), | |
| Krea2PrepareLatentsStep(), | |
| Krea2SetTimestepsStep(), | |
| Krea2RoPEInputsStep(), | |
| Krea2DenoiseStep(), | |
| Krea2AfterDenoiseStep(), | |
| ] | |
| block_names = [ | |
| "input", | |
| "prepare_latents", | |
| "set_timesteps", | |
| "prepare_rope_inputs", | |
| "denoise", | |
| "after_denoise", | |
| ] | |
| def description(self): | |
| return "step that denoise noise into image for text2image task. It includes the denoise loop, as well as prepare the inputs (timesteps, latents, rope inputs etc.)." | |
| def outputs(self): | |
| return [ | |
| OutputParam.template("latents"), | |
| ] | |
| # Krea 2 (inpainting) | |
| # auto_docstring | |
| class Krea2InpaintCoreDenoiseStep(SequentialPipelineBlocks): | |
| """ | |
| step that denoise noise into image for inpaint task. It includes the denoise loop, as well as prepare the inputs | |
| (timesteps, latents, rope inputs etc.). | |
| Components: | |
| pachifier (`Krea2Pachifier`) scheduler (`FlowMatchEulerDiscreteScheduler`) guider (`ClassifierFreeGuidance`) | |
| transformer (`Krea2Transformer2DModel`) | |
| Inputs: | |
| num_images_per_prompt (`int`, *optional*, defaults to 1): | |
| The number of images to generate per prompt. | |
| prompt_embeds (`Tensor`): | |
| text embeddings used to guide the image generation. Can be generated from text_encoder step. | |
| prompt_embeds_mask (`Tensor`): | |
| mask for the text embeddings. Can be generated from text_encoder step. | |
| negative_prompt_embeds (`Tensor`, *optional*): | |
| negative text embeddings used to guide the image generation. Can be generated from text_encoder step. | |
| negative_prompt_embeds_mask (`Tensor`, *optional*): | |
| mask for the negative text embeddings. Can be generated from text_encoder step. | |
| height (`int`, *optional*): | |
| The height in pixels of the generated image. | |
| width (`int`, *optional*): | |
| The width in pixels of the generated image. | |
| image_latents (`Tensor`, *optional*): | |
| image latents used to guide the image generation. Can be generated from vae_encoder step. | |
| processed_mask_image (`Tensor`, *optional*): | |
| The processed mask image | |
| latents (`Tensor`, *optional*): | |
| Pre-generated noisy latents for image generation. | |
| generator (`Generator`, *optional*): | |
| Torch generator for deterministic generation. | |
| num_inference_steps (`int`, *optional*, defaults to 28): | |
| The number of denoising steps. | |
| sigmas (`list`, *optional*): | |
| Custom sigmas for the denoising process. | |
| mu (`float`, *optional*): | |
| Fixed timestep shift for the scheduler. Pass `1.15` for the few-step distilled (TDM/turbo) checkpoint; if | |
| not provided, computed from the image sequence length (base checkpoint behavior). | |
| strength (`float`, *optional*, defaults to 0.9): | |
| Strength for img2img/inpainting. | |
| **denoiser_input_fields (`None`, *optional*): | |
| conditional model inputs for the denoiser: e.g. prompt_embeds, negative_prompt_embeds, etc. | |
| Outputs: | |
| latents (`Tensor`): | |
| Denoised latents. | |
| """ | |
| model_name = "krea2" | |
| block_classes = [ | |
| Krea2InpaintInputStep(), | |
| Krea2PrepareLatentsStep(), | |
| Krea2SetTimestepsWithStrengthStep(), | |
| Krea2InpaintPrepareLatentsStep(), | |
| Krea2RoPEInputsStep(), | |
| Krea2InpaintDenoiseStep(), | |
| Krea2AfterDenoiseStep(), | |
| ] | |
| block_names = [ | |
| "input", | |
| "prepare_latents", | |
| "set_timesteps", | |
| "prepare_inpaint_latents", | |
| "prepare_rope_inputs", | |
| "denoise", | |
| "after_denoise", | |
| ] | |
| def description(self): | |
| return "step that denoise noise into image for inpaint task. It includes the denoise loop, as well as prepare the inputs (timesteps, latents, rope inputs etc.)." | |
| def outputs(self): | |
| return [ | |
| OutputParam.template("latents"), | |
| ] | |
| # Krea 2 (image2image) | |
| # auto_docstring | |
| class Krea2Img2ImgCoreDenoiseStep(SequentialPipelineBlocks): | |
| """ | |
| step that denoise noise into image for img2img task. It includes the denoise loop, as well as prepare the inputs | |
| (timesteps, latents, rope inputs etc.). | |
| Components: | |
| pachifier (`Krea2Pachifier`) scheduler (`FlowMatchEulerDiscreteScheduler`) guider (`ClassifierFreeGuidance`) | |
| transformer (`Krea2Transformer2DModel`) | |
| Inputs: | |
| num_images_per_prompt (`int`, *optional*, defaults to 1): | |
| The number of images to generate per prompt. | |
| prompt_embeds (`Tensor`): | |
| text embeddings used to guide the image generation. Can be generated from text_encoder step. | |
| prompt_embeds_mask (`Tensor`): | |
| mask for the text embeddings. Can be generated from text_encoder step. | |
| negative_prompt_embeds (`Tensor`, *optional*): | |
| negative text embeddings used to guide the image generation. Can be generated from text_encoder step. | |
| negative_prompt_embeds_mask (`Tensor`, *optional*): | |
| mask for the negative text embeddings. Can be generated from text_encoder step. | |
| height (`int`, *optional*): | |
| The height in pixels of the generated image. | |
| width (`int`, *optional*): | |
| The width in pixels of the generated image. | |
| image_latents (`Tensor`): | |
| image latents used to guide the image generation. Can be generated from vae_encoder step. | |
| latents (`Tensor`, *optional*): | |
| Pre-generated noisy latents for image generation. | |
| generator (`Generator`, *optional*): | |
| Torch generator for deterministic generation. | |
| num_inference_steps (`int`, *optional*, defaults to 28): | |
| The number of denoising steps. | |
| sigmas (`list`, *optional*): | |
| Custom sigmas for the denoising process. | |
| mu (`float`, *optional*): | |
| Fixed timestep shift for the scheduler. Pass `1.15` for the few-step distilled (TDM/turbo) checkpoint; if | |
| not provided, computed from the image sequence length (base checkpoint behavior). | |
| strength (`float`, *optional*, defaults to 0.9): | |
| Strength for img2img/inpainting. | |
| **denoiser_input_fields (`None`, *optional*): | |
| conditional model inputs for the denoiser: e.g. prompt_embeds, negative_prompt_embeds, etc. | |
| Outputs: | |
| latents (`Tensor`): | |
| Denoised latents. | |
| """ | |
| model_name = "krea2" | |
| block_classes = [ | |
| Krea2Img2ImgInputStep(), | |
| Krea2PrepareLatentsStep(), | |
| Krea2SetTimestepsWithStrengthStep(), | |
| Krea2PrepareLatentsWithStrengthStep(), | |
| Krea2RoPEInputsStep(), | |
| Krea2DenoiseStep(), | |
| Krea2AfterDenoiseStep(), | |
| ] | |
| block_names = [ | |
| "input", | |
| "prepare_latents", | |
| "set_timesteps", | |
| "prepare_img2img_latents", | |
| "prepare_rope_inputs", | |
| "denoise", | |
| "after_denoise", | |
| ] | |
| def description(self): | |
| return "step that denoise noise into image for img2img task. It includes the denoise loop, as well as prepare the inputs (timesteps, latents, rope inputs etc.)." | |
| def outputs(self): | |
| return [ | |
| OutputParam.template("latents"), | |
| ] | |
| # Auto denoise step for Krea 2 | |
| class Krea2AutoCoreDenoiseStep(AutoPipelineBlocks): | |
| model_name = "krea2" | |
| block_classes = [ | |
| Krea2InpaintCoreDenoiseStep, | |
| Krea2Img2ImgCoreDenoiseStep, | |
| Krea2CoreDenoiseStep, | |
| ] | |
| block_names = [ | |
| "inpaint", | |
| "img2img", | |
| "text2image", | |
| ] | |
| block_trigger_inputs = ["processed_mask_image", "image_latents", None] | |
| def description(self): | |
| return ( | |
| "Core step that performs the denoising process. \n" | |
| + " - `Krea2InpaintCoreDenoiseStep` (inpaint) is used when `processed_mask_image` is provided.\n" | |
| + " - `Krea2Img2ImgCoreDenoiseStep` (img2img) is used when `image_latents` is provided.\n" | |
| + " - `Krea2CoreDenoiseStep` (text2image) is used otherwise.\n" | |
| + "This step support text-to-image, image-to-image, and inpainting tasks for Krea 2:\n" | |
| + " - for image-to-image generation, you need to provide `image_latents`\n" | |
| + " - for inpainting, you need to provide `processed_mask_image` and `image_latents`\n" | |
| + " - for text-to-image generation, all you need to provide is prompt embeddings" | |
| ) | |
| def outputs(self): | |
| return [ | |
| OutputParam.template("latents"), | |
| ] | |
| # ==================== | |
| # 4. DECODE | |
| # ==================== | |
| # standard decode step works for most tasks except for inpaint | |
| # auto_docstring | |
| class Krea2DecodeStep(SequentialPipelineBlocks): | |
| """ | |
| Decode step that decodes the latents to images and postprocess the generated image. | |
| Components: | |
| vae (`AutoencoderKLQwenImage`) image_processor (`VaeImageProcessor`) | |
| Inputs: | |
| latents (`Tensor`): | |
| The denoised latents to decode, can be generated in the denoise step and unpacked in the after denoise | |
| step. | |
| output_type (`str`, *optional*, defaults to pil): | |
| Output format: 'pil', 'np', 'pt'. | |
| Outputs: | |
| images (`list`): | |
| Generated images. (tensor output of the vae decoder.) | |
| """ | |
| model_name = "krea2" | |
| block_classes = [Krea2DecoderStep(), Krea2ProcessImagesOutputStep()] | |
| block_names = ["decode", "postprocess"] | |
| def description(self): | |
| return "Decode step that decodes the latents to images and postprocess the generated image." | |
| # Inpaint decode step | |
| # auto_docstring | |
| class Krea2InpaintDecodeStep(SequentialPipelineBlocks): | |
| """ | |
| Decode step that decodes the latents to images and postprocess the generated image, optionally apply the mask | |
| overlay to the original image. | |
| Components: | |
| vae (`AutoencoderKLQwenImage`) image_mask_processor (`InpaintProcessor`) | |
| Inputs: | |
| latents (`Tensor`): | |
| The denoised latents to decode, can be generated in the denoise step and unpacked in the after denoise | |
| step. | |
| output_type (`str`, *optional*, defaults to pil): | |
| Output format: 'pil', 'np', 'pt'. | |
| mask_overlay_kwargs (`dict`, *optional*): | |
| The kwargs for the postprocess step to apply the mask overlay. generated in | |
| Krea2InpaintProcessImagesInputStep. | |
| Outputs: | |
| images (`list`): | |
| Generated images. (tensor output of the vae decoder.) | |
| """ | |
| model_name = "krea2" | |
| block_classes = [Krea2DecoderStep(), Krea2InpaintProcessImagesOutputStep()] | |
| block_names = ["decode", "postprocess"] | |
| def description(self): | |
| return "Decode step that decodes the latents to images and postprocess the generated image, optionally apply the mask overlay to the original image." | |
| # Auto decode step for Krea 2 | |
| class Krea2AutoDecodeStep(AutoPipelineBlocks): | |
| model_name = "krea2" | |
| block_classes = [Krea2InpaintDecodeStep, Krea2DecodeStep] | |
| block_names = ["inpaint_decode", "decode"] | |
| block_trigger_inputs = ["mask", None] | |
| def description(self): | |
| return ( | |
| "Decode step that decode the latents into images. \n" | |
| " This is an auto pipeline block that works for inpaint/text2image/img2img tasks.\n" | |
| + " - `Krea2InpaintDecodeStep` (inpaint_decode) is used when `mask` is provided.\n" | |
| + " - `Krea2DecodeStep` (decode) is used when `mask` is not provided.\n" | |
| ) | |
| # ==================== | |
| # 5. AUTO BLOCKS & PRESETS | |
| # ==================== | |
| AUTO_BLOCKS = InsertableDict( | |
| [ | |
| ("text_encoder", Krea2AutoTextEncoderStep()), | |
| ("vae_encoder", Krea2AutoVaeEncoderStep()), | |
| ("denoise", Krea2AutoCoreDenoiseStep()), | |
| ("decode", Krea2AutoDecodeStep()), | |
| ] | |
| ) | |
| # auto_docstring | |
| class Krea2AutoBlocks(SequentialPipelineBlocks): | |
| """ | |
| Auto Modular pipeline for text-to-image, image-to-image, and inpainting tasks using Krea 2. | |
| Supported workflows: | |
| - `text2image`: requires `prompt` | |
| - `image2image`: requires `prompt`, `image` | |
| - `inpainting`: requires `prompt`, `mask_image`, `image` | |
| Components: | |
| text_encoder (`Qwen3VLModel`): The text encoder to use tokenizer (`Qwen2Tokenizer`): The tokenizer to use | |
| guider (`ClassifierFreeGuidance`) image_mask_processor (`InpaintProcessor`) vae (`AutoencoderKLQwenImage`) | |
| image_processor (`VaeImageProcessor`) pachifier (`Krea2Pachifier`) scheduler | |
| (`FlowMatchEulerDiscreteScheduler`) transformer (`Krea2Transformer2DModel`) | |
| Inputs: | |
| prompt (`str`, *optional*): | |
| The prompt or prompts to guide image generation. | |
| negative_prompt (`str`, *optional*): | |
| The prompt or prompts not to guide the image generation. | |
| max_sequence_length (`int`, *optional*, defaults to 512): | |
| Maximum sequence length for prompt encoding. | |
| mask_image (`Image`, *optional*): | |
| Mask image for inpainting. | |
| image (`Image | list`, *optional*): | |
| Reference image(s) for denoising. Can be a single image or list of images. | |
| height (`int`, *optional*): | |
| The height in pixels of the generated image. | |
| width (`int`, *optional*): | |
| The width in pixels of the generated image. | |
| padding_mask_crop (`int`, *optional*): | |
| Padding for mask cropping in inpainting. | |
| generator (`Generator`, *optional*): | |
| Torch generator for deterministic generation. | |
| num_images_per_prompt (`int`, *optional*, defaults to 1): | |
| The number of images to generate per prompt. | |
| prompt_embeds (`Tensor`): | |
| text embeddings used to guide the image generation. Can be generated from text_encoder step. | |
| prompt_embeds_mask (`Tensor`): | |
| mask for the text embeddings. Can be generated from text_encoder step. | |
| negative_prompt_embeds (`Tensor`, *optional*): | |
| negative text embeddings used to guide the image generation. Can be generated from text_encoder step. | |
| negative_prompt_embeds_mask (`Tensor`, *optional*): | |
| mask for the negative text embeddings. Can be generated from text_encoder step. | |
| image_latents (`Tensor`, *optional*): | |
| image latents used to guide the image generation. Can be generated from vae_encoder step. | |
| processed_mask_image (`Tensor`, *optional*): | |
| The processed mask image | |
| latents (`Tensor`): | |
| Pre-generated noisy latents for image generation. | |
| num_inference_steps (`int`): | |
| The number of denoising steps. | |
| sigmas (`list`, *optional*): | |
| Custom sigmas for the denoising process. | |
| mu (`float`, *optional*): | |
| Fixed timestep shift for the scheduler. Pass `1.15` for the few-step distilled (TDM/turbo) checkpoint; if | |
| not provided, computed from the image sequence length (base checkpoint behavior). | |
| strength (`float`, *optional*, defaults to 0.9): | |
| Strength for img2img/inpainting. | |
| **denoiser_input_fields (`None`, *optional*): | |
| conditional model inputs for the denoiser: e.g. prompt_embeds, negative_prompt_embeds, etc. | |
| output_type (`str`, *optional*, defaults to pil): | |
| Output format: 'pil', 'np', 'pt'. | |
| mask_overlay_kwargs (`dict`, *optional*): | |
| The kwargs for the postprocess step to apply the mask overlay. generated in | |
| Krea2InpaintProcessImagesInputStep. | |
| Outputs: | |
| images (`list`): | |
| Generated images. | |
| """ | |
| model_name = "krea2" | |
| block_classes = AUTO_BLOCKS.values() | |
| block_names = AUTO_BLOCKS.keys() | |
| # Workflow map defines the trigger conditions for each workflow. | |
| # How to define: | |
| # - Only include required inputs and trigger inputs (inputs that determine which blocks run) | |
| # - currently, only supports `True` means the workflow triggers when the input is not None | |
| _workflow_map = { | |
| "text2image": {"prompt": True}, | |
| "image2image": {"prompt": True, "image": True}, | |
| "inpainting": {"prompt": True, "mask_image": True, "image": True}, | |
| } | |
| def description(self): | |
| return "Auto Modular pipeline for text-to-image, image-to-image, and inpainting tasks using Krea 2." | |
| def outputs(self): | |
| return [OutputParam.template("images")] | |