ComfyUI-Advanced-ControlNet/adv_control/nodes_loosecontrol.py

import folder_paths
import comfy.utils
import comfy.model_detection
import comfy.model_management
import comfy.lora
from comfy.model_patcher import ModelPatcher

from .utils import TimestepKeyframeGroup
from .control import ControlNetAdvanced, load_controlnet




def convert_cn_lora_from_diffusers(cn_model: ModelPatcher, lora_path: str):
    lora_data = comfy.utils.load_torch_file(lora_path, safe_load=True)
    unet_dtype = comfy.model_management.unet_dtype()
    for key, value in lora_data.items():
        lora_data[key] = value.to(unet_dtype)
    diffusers_keys = comfy.utils.unet_to_diffusers(cn_model.model.state_dict())

    #lora_data = comfy.model_detection.unet_config_from_diffusers_unet(lora_data, dtype=unet_dtype)



    #key_map = comfy.lora.model_lora_keys_unet(cn_model.model, key_map)
    lora_data = comfy.lora.load_lora(lora_data, to_load=diffusers_keys)

    # TODO: detect if diffusers for sure? not sure if needed at this time, since cn loras are
    # only used currently for LOOSEControl, and those are all in diffusers format
    #unet_dtype = comfy.model_management.unet_dtype()
    #lora_data = comfy.model_detection.unet_config_from_diffusers_unet(lora_data, unet_dtype)
    return lora_data


class ControlNetLoaderWithLoraAdvanced:
    @classmethod
    def INPUT_TYPES(s):
        return {
            "required": {
                "control_net_name": (folder_paths.get_filename_list("controlnet"), ),
                "cn_lora_name": (folder_paths.get_filename_list("controlnet"), ),
                "cn_lora_strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001}),
            },
            "optional": {
                "timestep_keyframe": ("TIMESTEP_KEYFRAME", ),
            }
        }

    RETURN_TYPES = ("CONTROL_NET", )
    FUNCTION = "load_controlnet"

    CATEGORY = "Adv-ControlNet 🛂🅐🅒🅝/LOOSEControl"

    def load_controlnet(self, control_net_name, cn_lora_name, cn_lora_strength: float,
                        timestep_keyframe: TimestepKeyframeGroup=None
                        ):
        controlnet_path = folder_paths.get_full_path("controlnet", control_net_name)
        controlnet: ControlNetAdvanced = load_controlnet(controlnet_path, timestep_keyframe)
        if not isinstance(controlnet, ControlNetAdvanced):
            raise ValueError("Type {} is not compatible with CN LoRA features at this time.")
        # now, try to load CN LoRA
        lora_path = folder_paths.get_full_path("controlnet", cn_lora_name)
        lora_data = convert_cn_lora_from_diffusers(cn_model=controlnet.control_model_wrapped, lora_path=lora_path)
        # apply patches to wrapped control_model
        controlnet.control_model_wrapped.add_patches(lora_data, strength_patch=cn_lora_strength)
        # all done
        return (controlnet,)