# Copyright 2026 Krea AI and The HuggingFace Team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import Any import torch from diffusers.configuration_utils import FrozenDict from diffusers.image_processor import InpaintProcessor, VaeImageProcessor from diffusers.models import AutoencoderKLQwenImage from diffusers.utils import logging from diffusers.modular_pipelines.modular_pipeline import ModularPipelineBlocks, PipelineState from diffusers.modular_pipelines.modular_pipeline_utils import ComponentSpec, InputParam, OutputParam from .modular_pipeline import Krea2ModularPipeline, Krea2Pachifier logger = logging.get_logger(__name__) # after denoising loop (unpack latents) class Krea2AfterDenoiseStep(ModularPipelineBlocks): model_name = "krea2" @property def description(self) -> str: return "Step that unpack the latents from 3D tensor (batch_size, sequence_length, channels) into 5D tensor (batch_size, channels, 1, height, width)" @property def expected_components(self) -> list[ComponentSpec]: components = [ ComponentSpec("pachifier", Krea2Pachifier, default_creation_method="from_config"), ] return components @property def inputs(self) -> list[InputParam]: return [ InputParam.template("height", required=True), InputParam.template("width", required=True), InputParam( name="latents", required=True, type_hint=torch.Tensor, description="The latents to decode, can be generated in the denoise step.", ), ] @property def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam( name="latents", type_hint=torch.Tensor, description="The denoised latents unpacked to B, C, 1, H, W" ), ] @torch.no_grad() def __call__(self, components: Krea2ModularPipeline, state: PipelineState) -> PipelineState: block_state = self.get_block_state(state) vae_scale_factor = components.vae_scale_factor block_state.latents = components.pachifier.unpack_latents( block_state.latents, block_state.height, block_state.width, vae_scale_factor=vae_scale_factor ) self.set_block_state(state, block_state) return components, state # decode step class Krea2DecoderStep(ModularPipelineBlocks): model_name = "krea2" @property def description(self) -> str: return "Step that decodes the latents to images" @property def expected_components(self) -> list[ComponentSpec]: components = [ ComponentSpec("vae", AutoencoderKLQwenImage), ] return components @property def inputs(self) -> list[InputParam]: return [ InputParam( name="latents", required=True, type_hint=torch.Tensor, description="The denoised latents to decode, can be generated in the denoise step and unpacked in the after denoise step.", ), ] @property def intermediate_outputs(self) -> list[OutputParam]: return [OutputParam.template("images", note="tensor output of the vae decoder.")] @torch.no_grad() def __call__(self, components: Krea2ModularPipeline, state: PipelineState) -> PipelineState: block_state = self.get_block_state(state) if block_state.latents.ndim == 4: block_state.latents = block_state.latents.unsqueeze(dim=1) elif block_state.latents.ndim != 5: raise ValueError( f"expect latents to be a 4D or 5D tensor but got: {block_state.latents.shape}. Please make sure the latents are unpacked before decode step." ) block_state.latents = block_state.latents.to(components.vae.dtype) latents_mean = ( torch.tensor(components.vae.config.latents_mean) .view(1, components.vae.config.z_dim, 1, 1, 1) .to(block_state.latents.device, block_state.latents.dtype) ) latents_std = 1.0 / torch.tensor(components.vae.config.latents_std).view( 1, components.vae.config.z_dim, 1, 1, 1 ).to(block_state.latents.device, block_state.latents.dtype) block_state.latents = block_state.latents / latents_std + latents_mean block_state.images = components.vae.decode(block_state.latents, return_dict=False)[0][:, :, 0] self.set_block_state(state, block_state) return components, state # postprocess the decoded images class Krea2ProcessImagesOutputStep(ModularPipelineBlocks): model_name = "krea2" @property def description(self) -> str: return "postprocess the generated image" @property def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec( "image_processor", VaeImageProcessor, config=FrozenDict({"vae_scale_factor": 16}), default_creation_method="from_config", ), ] @property def inputs(self) -> list[InputParam]: return [ InputParam( name="images", required=True, type_hint=torch.Tensor, description="the generated image tensor from decoders step", ), InputParam.template("output_type"), ] @property def intermediate_outputs(self) -> list[OutputParam]: return [OutputParam.template("images")] @staticmethod def check_inputs(output_type): if output_type not in ["pil", "np", "pt"]: raise ValueError(f"Invalid output_type: {output_type}") @torch.no_grad() def __call__(self, components: Krea2ModularPipeline, state: PipelineState): block_state = self.get_block_state(state) self.check_inputs(block_state.output_type) block_state.images = components.image_processor.postprocess( image=block_state.images, output_type=block_state.output_type, ) self.set_block_state(state, block_state) return components, state class Krea2InpaintProcessImagesOutputStep(ModularPipelineBlocks): model_name = "krea2" @property def description(self) -> str: return "postprocess the generated image, optionally apply the mask overlay to the original image." @property def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec( "image_mask_processor", InpaintProcessor, config=FrozenDict({"vae_scale_factor": 16}), default_creation_method="from_config", ), ] @property def inputs(self) -> list[InputParam]: return [ InputParam( name="images", required=True, type_hint=torch.Tensor, description="the generated image tensor from decoders step", ), InputParam.template("output_type"), InputParam( name="mask_overlay_kwargs", type_hint=dict[str, Any], description="The kwargs for the postprocess step to apply the mask overlay. generated in Krea2InpaintProcessImagesInputStep.", ), ] @property def intermediate_outputs(self) -> list[OutputParam]: return [OutputParam.template("images")] @staticmethod def check_inputs(output_type, mask_overlay_kwargs): if output_type not in ["pil", "np", "pt"]: raise ValueError(f"Invalid output_type: {output_type}") if mask_overlay_kwargs and output_type != "pil": raise ValueError("only support output_type 'pil' for mask overlay") @torch.no_grad() def __call__(self, components: Krea2ModularPipeline, state: PipelineState): block_state = self.get_block_state(state) self.check_inputs(block_state.output_type, block_state.mask_overlay_kwargs) if block_state.mask_overlay_kwargs is None: mask_overlay_kwargs = {} else: mask_overlay_kwargs = block_state.mask_overlay_kwargs block_state.images = components.image_mask_processor.postprocess( image=block_state.images, **mask_overlay_kwargs, ) self.set_block_state(state, block_state) return components, state