Upload folder using huggingface_hub (part 5)

.gitattributes

@@ -34,3 +34,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 .github/workflows/logo.gif filter=lfs diff=lfs merge=lfs -text
+models/train/MiniMax-H3-Ref2VA-CineDance-filtered-multishot-native-partial/wandb_log/wandb/run-20260916_162111-qrso8o89/run-qrso8o89.wandb filter=lfs diff=lfs merge=lfs -text
+models/train/MiniMax-H3-Ref2VA-CineDance-multishot-full-v2/wandb_log/wandb/run-20260831_033451-73yd8zqv/run-73yd8zqv.wandb filter=lfs diff=lfs merge=lfs -text

examples/qwen_image/model_training/validate_lora/Qwen-Image-Edit-2511.py

@@ -0,0 +1,24 @@
+import torch
+from PIL import Image
+from diffsynth.pipelines.qwen_image import QwenImagePipeline, ModelConfig
+
+pipe = QwenImagePipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="Qwen/Qwen-Image-Edit-2511", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"),
+    ],
+    tokenizer_config=None,
+    processor_config=ModelConfig(model_id="Qwen/Qwen-Image-Edit", origin_file_pattern="processor/"),
+)
+pipe.load_lora(pipe.dit, "models/train/Qwen-Image-Edit-2511_lora/epoch-4.safetensors")
+
+prompt = "Change the color of the dress in Figure 1 to the color shown in Figure 2."
+images = [
+    Image.open("data/example_image_dataset/edit/image1.jpg").resize((1024, 1024)),
+    Image.open("data/example_image_dataset/edit/image_color.jpg").resize((1024, 1024)),
+]
+image = pipe(prompt, edit_image=images, seed=123, num_inference_steps=40, height=1024, width=1024, zero_cond_t=True)
+image.save("image.jpg")

examples/qwen_image/model_training/validate_lora/Qwen-Image-Edit.py

@@ -0,0 +1,21 @@
+import torch
+from PIL import Image
+from diffsynth.pipelines.qwen_image import QwenImagePipeline, ModelConfig
+
+pipe = QwenImagePipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="Qwen/Qwen-Image-Edit", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"),
+    ],
+    tokenizer_config=None,
+    processor_config=ModelConfig(model_id="Qwen/Qwen-Image-Edit", origin_file_pattern="processor/"),
+)
+pipe.load_lora(pipe.dit, "models/train/Qwen-Image-Edit_lora/epoch-4.safetensors")
+
+prompt = "将裙子改为粉色"
+image = Image.open("data/example_image_dataset/edit/image1.jpg").resize((1024, 1024))
+image = pipe(prompt, edit_image=image, seed=0, num_inference_steps=40, height=1024, width=1024)
+image.save(f"image.jpg")

examples/qwen_image/model_training/validate_lora/Qwen-Image-EliGen-Poster.py

@@ -0,0 +1,29 @@
+from diffsynth.pipelines.qwen_image import QwenImagePipeline, ModelConfig
+import torch
+from PIL import Image
+
+
+pipe = QwenImagePipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"),
+    ],
+    tokenizer_config=ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="tokenizer/"),
+)
+pipe.load_lora(pipe.dit, "models/train/Qwen-Image-EliGen-Poster_lora/epoch-4.safetensors")
+
+
+entity_prompts = ["A beautiful girl", "sign 'Entity Control'", "shorts", "shirt"]
+global_prompt = "A beautiful girl wearing shirt and shorts in the street,  holding a sign 'Entity Control'"
+masks = [Image.open(f"data/example_image_dataset/eligen/{i}.png").convert('RGB') for i in range(len(entity_prompts))]
+
+image = pipe(global_prompt,
+             seed=0,
+             height=1024,
+             width=1024,
+             eligen_entity_prompts=entity_prompts,
+             eligen_entity_masks=masks)
+image.save("Qwen-Image-EliGen-Poster.jpg")

examples/qwen_image/model_training/validate_lora/Qwen-Image-EliGen.py

@@ -0,0 +1,29 @@
+from diffsynth.pipelines.qwen_image import QwenImagePipeline, ModelConfig
+import torch
+from PIL import Image
+
+
+pipe = QwenImagePipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"),
+    ],
+    tokenizer_config=ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="tokenizer/"),
+)
+pipe.load_lora(pipe.dit, "models/train/Qwen-Image-EliGen_lora/epoch-4.safetensors")
+
+
+entity_prompts = ["A beautiful girl", "sign 'Entity Control'", "shorts", "shirt"]
+global_prompt = "A beautiful girl wearing shirt and shorts in the street,  holding a sign 'Entity Control'"
+masks = [Image.open(f"data/example_image_dataset/eligen/{i}.png").convert('RGB') for i in range(len(entity_prompts))]
+
+image = pipe(global_prompt,
+             seed=0,
+             height=1024,
+             width=1024,
+             eligen_entity_prompts=entity_prompts,
+             eligen_entity_masks=masks)
+image.save("Qwen-Image_EliGen.jpg")

examples/qwen_image/model_training/validate_lora/Qwen-Image-In-Context-Control-Union.py

@@ -0,0 +1,19 @@
+from PIL import Image
+import torch
+from diffsynth.pipelines.qwen_image import QwenImagePipeline, ModelConfig
+
+pipe = QwenImagePipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"),
+    ],
+    tokenizer_config=ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="tokenizer/"),
+)
+pipe.load_lora(pipe.dit, "models/train/Qwen-Image-In-Context-Control-Union_lora/epoch-4.safetensors")
+image = Image.open("data/example_image_dataset/canny/image_1.jpg").resize((1024, 1024))
+prompt = "Context_Control. a dog"
+image = pipe(prompt=prompt, seed=0, context_image=image, height=1024, width=1024)
+image.save("image_context.jpg")

examples/qwen_image/model_training/validate_lora/Qwen-Image-Layered-Control-V2.py

@@ -0,0 +1,37 @@
+from diffsynth.pipelines.qwen_image import QwenImagePipeline, ModelConfig
+from modelscope import dataset_snapshot_download
+from PIL import Image
+import torch
+
+pipe = QwenImagePipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="DiffSynth-Studio/Qwen-Image-Layered-Control", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image-Layered", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"),
+    ],
+    tokenizer_config=ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="tokenizer/"),
+)
+pipe.load_lora(pipe.dit, "models/train/Qwen-Image-Layered-Control-V2_lora/epoch-4.safetensors")
+
+prompt = "Text 'APRIL'"
+input_image = Image.open("data/example_image_dataset/layer_v2/image_1.png").convert("RGBA").resize((1024, 1024))
+image = pipe(
+    prompt, seed=0,
+    height=1024, width=1024,
+    layer_input_image=input_image, layer_num=0,
+    num_inference_steps=10, cfg_scale=4,
+)
+image[0].save("image_prompt.png")
+
+mask_image = Image.open("data/example_image_dataset/layer_v2/mask_2.png").convert("RGBA").resize((1024, 1024))
+input_image = Image.open("data/example_image_dataset/layer_v2/image_2.png").convert("RGBA").resize((1024, 1024))
+image = pipe(
+    prompt, seed=0,
+    height=1024, width=1024,
+    layer_input_image=input_image, layer_num=0,
+    context_image=mask_image,
+    num_inference_steps=10, cfg_scale=1.0,
+)
+image[0].save("image_mask.png")

examples/qwen_image/model_training/validate_lora/Qwen-Image-Layered-Control.py

@@ -0,0 +1,25 @@
+from diffsynth.pipelines.qwen_image import QwenImagePipeline, ModelConfig
+from diffsynth import load_state_dict
+from PIL import Image
+import torch
+
+
+pipe = QwenImagePipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="DiffSynth-Studio/Qwen-Image-Layered-Control", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image-Layered", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"),
+    ],
+    tokenizer_config=ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="tokenizer/"),
+)
+pipe.load_lora(pipe.dit, "models/train/Qwen-Image-Layered-Control_lora/epoch-4.safetensors")
+prompt = "Text 'HELLO' and 'Have a great day'"
+input_image = Image.open("data/example_image_dataset/layer/image.png").convert("RGBA").resize((864, 480))
+images = pipe(
+    prompt, seed=0,
+    height=480, width=864,
+    layer_input_image=input_image, layer_num=0,
+)
+images[0].save("image.png")

examples/qwen_image/model_training/validate_lora/Qwen-Image-Layered.py

@@ -0,0 +1,27 @@
+from diffsynth.pipelines.qwen_image import QwenImagePipeline, ModelConfig
+from diffsynth import load_state_dict
+from PIL import Image
+import torch
+
+
+pipe = QwenImagePipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="Qwen/Qwen-Image-Layered", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image-Layered", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"),
+    ],
+    tokenizer_config=ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="tokenizer/"),
+)
+pipe.load_lora(pipe.dit, "models/train/Qwen-Image-Layered_lora/epoch-4.safetensors")
+prompt = "a poster"
+input_image = Image.open("data/example_image_dataset/layer/image.png").convert("RGBA").resize((864, 480))
+images = pipe(
+    prompt, seed=0,
+    height=480, width=864,
+    layer_input_image=input_image, layer_num=3,
+)
+for i, image in enumerate(images):
+    if i == 0: continue # The first image is the input image.
+    image.save(f"image_{i}.png")

examples/qwen_image/model_training/validate_lora/Qwen-Image.py

@@ -0,0 +1,18 @@
+from diffsynth.pipelines.qwen_image import QwenImagePipeline, ModelConfig
+import torch
+
+
+pipe = QwenImagePipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"),
+    ],
+    tokenizer_config=ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="tokenizer/"),
+)
+pipe.load_lora(pipe.dit, "models/train/Qwen-Image_lora/epoch-4.safetensors")
+prompt = "a dog"
+image = pipe(prompt, seed=0)
+image.save("image.jpg")

examples/qwen_video_edit/model_inference/Qwen-Video-Edit.py

@@ -0,0 +1,27 @@
+import torch
+from modelscope import dataset_snapshot_download
+from diffsynth.core import ModelConfig
+from diffsynth.pipelines.qwen_video_edit import QwenVideoEditPipeline
+from diffsynth.utils.data import VideoData, save_video
+
+dataset_snapshot_download(
+    "DiffSynth-Studio/diffsynth_example_dataset",
+    local_dir="data/diffsynth_example_dataset",
+    allow_file_pattern="qwen_video_edit/Qwen-Video-Edit/*"
+)
+
+edit_video = VideoData("data/diffsynth_example_dataset/qwen_video_edit/Qwen-Video-Edit/source.mp4")
+prompts = [
+    "Transform the video into Japanese anime style",
+]
+pipe = QwenVideoEditPipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="yunpeng1998/Qwen-Video-Edit", origin_file_pattern="360P/step-30000.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors"),
+        ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="Wan2.1_VAE.pth"),
+    ],
+)
+video = pipe(edit_video=edit_video, prompts=prompts, height=640, width=384, num_frames=45, cfg_scale=4.0, num_inference_steps=40, seed=0)
+save_video(video, "video_Qwen-Video-Edit.mp4", fps=16)

examples/qwen_video_edit/model_inference_low_vram/Qwen-Video-Edit.py

@@ -0,0 +1,40 @@
+import torch
+from modelscope import dataset_snapshot_download
+from diffsynth.core import ModelConfig
+from diffsynth.pipelines.qwen_video_edit import QwenVideoEditPipeline
+from diffsynth.utils.data import VideoData, save_video
+
+vram_config = {
+    "offload_dtype": torch.bfloat16,
+    "offload_device": "cpu",
+    "onload_dtype": torch.bfloat16,
+    "onload_device": "cpu",
+    "preparing_dtype": torch.bfloat16,
+    "preparing_device": "cuda",
+    "computation_dtype": torch.bfloat16,
+    "computation_device": "cuda",
+}
+
+dataset_snapshot_download(
+    "DiffSynth-Studio/diffsynth_example_dataset",
+    local_dir="data/diffsynth_example_dataset",
+    allow_file_pattern="qwen_video_edit/Qwen-Video-Edit/*"
+)
+
+edit_video = VideoData("data/diffsynth_example_dataset/qwen_video_edit/Qwen-Video-Edit/source.mp4")
+
+prompts = [
+    "Transform the video into Japanese anime style",
+]
+pipe = QwenVideoEditPipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="yunpeng1998/Qwen-Video-Edit", origin_file_pattern="360P/step-30000.safetensors", **vram_config),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors", **vram_config),
+        ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config),
+    ],
+    vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5,
+)
+video = pipe(edit_video=edit_video, prompts=prompts, height=640, width=384, num_frames=45, cfg_scale=4.0, num_inference_steps=40, seed=0)
+save_video(video, "video_Qwen-Video-Edit.mp4", fps=16)

examples/qwen_video_edit/model_training/full/Qwen-Video-Edit.sh

@@ -0,0 +1,19 @@
+modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "qwen_video_edit/Qwen-Video-Edit/*" --local_dir ./data/diffsynth_example_dataset
+
+accelerate launch examples/qwen_video_edit/model_training/train.py \
+  --dataset_base_path data/diffsynth_example_dataset/qwen_video_edit/Qwen-Video-Edit \
+  --dataset_metadata_path data/diffsynth_example_dataset/qwen_video_edit/Qwen-Video-Edit/metadata.json \
+  --data_file_keys "video,input_video" \
+  --height 640 \
+  --width 384 \
+  --num_frames 45 \
+  --dataset_repeat 50 \
+  --model_id_with_origin_paths "yunpeng1998/Qwen-Video-Edit:360P/step-30000.safetensors,Qwen/Qwen-Image:text_encoder/model*.safetensors,Wan-AI/Wan2.1-T2V-1.3B:Wan2.1_VAE.pth" \
+  --learning_rate 1e-5 \
+  --num_epochs 2 \
+  --remove_prefix_in_ckpt "pipe.dit." \
+  --output_path "./models/train/Qwen-Video-Edit_full" \
+  --trainable_models "dit" \
+  --use_gradient_checkpointing \
+  --zero_cond_t \
+  --find_unused_parameters

examples/qwen_video_edit/model_training/full/accelerate_config_zero3.yaml

@@ -0,0 +1,23 @@
+compute_environment: LOCAL_MACHINE
+debug: false
+deepspeed_config:
+  gradient_accumulation_steps: 1
+  offload_optimizer_device: none
+  offload_param_device: none
+  zero3_init_flag: true
+  zero3_save_16bit_model: true
+  zero_stage: 3
+distributed_type: DEEPSPEED
+downcast_bf16: 'no'
+enable_cpu_affinity: false
+machine_rank: 0
+main_training_function: main
+mixed_precision: bf16
+num_machines: 1
+num_processes: 8
+rdzv_backend: static
+same_network: true
+tpu_env: []
+tpu_use_cluster: false
+tpu_use_sudo: false
+use_cpu: false

examples/qwen_video_edit/model_training/lora/Qwen-Video-Edit.sh

@@ -0,0 +1,22 @@
+modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "qwen_video_edit/Qwen-Video-Edit/*" --local_dir ./data/diffsynth_example_dataset
+
+accelerate launch examples/qwen_video_edit/model_training/train.py \
+  --dataset_base_path data/diffsynth_example_dataset/qwen_video_edit/Qwen-Video-Edit \
+  --dataset_metadata_path data/diffsynth_example_dataset/qwen_video_edit/Qwen-Video-Edit/metadata.json \
+  --data_file_keys "video,input_video" \
+  --height 640 \
+  --width 384 \
+  --num_frames 45 \
+  --dataset_repeat 50 \
+  --model_id_with_origin_paths "yunpeng1998/Qwen-Video-Edit:360P/step-30000.safetensors,Qwen/Qwen-Image:text_encoder/model*.safetensors,Wan-AI/Wan2.1-T2V-1.3B:Wan2.1_VAE.pth" \
+  --learning_rate 1e-4 \
+  --num_epochs 5 \
+  --remove_prefix_in_ckpt "pipe.dit." \
+  --output_path "./models/train/Qwen-Video-Edit_lora" \
+  --lora_base_model "dit" \
+  --lora_target_modules "to_q,to_k,to_v,add_q_proj,add_k_proj,add_v_proj,to_out.0,to_add_out,img_mlp.net.2,img_mod.1,txt_mlp.net.2,txt_mod.1" \
+  --lora_rank 32 \
+  --use_gradient_checkpointing \
+  --zero_cond_t \
+  --dataset_num_workers 8 \
+  --find_unused_parameters

examples/qwen_video_edit/model_training/train.py

@@ -0,0 +1,175 @@
+import torch, os, argparse, accelerate, warnings
+from diffsynth.core import UnifiedDataset, ModelConfig
+from diffsynth.pipelines.qwen_video_edit import QwenVideoEditPipeline
+from diffsynth.diffusion import *
+from diffsynth.core.data.operators import *
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+
+class QwenVideoEditTrainingModule(DiffusionTrainingModule):
+    def __init__(
+        self,
+        model_paths=None, model_id_with_origin_paths=None,
+        tokenizer_path=None, processor_path=None,
+        trainable_models=None,
+        lora_base_model=None, lora_target_modules="", lora_rank=32, lora_checkpoint=None,
+        preset_lora_path=None, preset_lora_model=None,
+        use_gradient_checkpointing=True,
+        use_gradient_checkpointing_offload=False,
+        extra_inputs=None,
+        fp8_models=None,
+        offload_models=None,
+        quant_options=None,
+        resume_from_checkpoint=None, remove_prefix_in_ckpt=None,
+        device="cpu",
+        task="sft",
+        zero_cond_t=False,
+        max_timestep_boundary=1.0,
+        min_timestep_boundary=0.0,
+    ):
+        super().__init__()
+        # Load models
+        model_configs = self.parse_model_configs(model_paths, model_id_with_origin_paths, fp8_models=fp8_models, offload_models=offload_models, quant_options=quant_options, device=device)
+        tokenizer_config = ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="tokenizer/") if tokenizer_path is None else ModelConfig(tokenizer_path)
+        processor_config = ModelConfig(model_id="Qwen/Qwen-Image-Edit", origin_file_pattern="processor/") if processor_path is None else ModelConfig(processor_path)
+        self.pipe = QwenVideoEditPipeline.from_pretrained(torch_dtype=torch.bfloat16, device=device, model_configs=model_configs, tokenizer_config=tokenizer_config, processor_config=processor_config)
+        self.pipe = self.split_pipeline_units(task, self.pipe, trainable_models, lora_base_model)
+        self.resume_from_checkpoint(resume_from_checkpoint, remove_prefix_in_ckpt)
+
+        # Training mode
+        self.switch_pipe_to_training_mode(
+            self.pipe, trainable_models,
+            lora_base_model, lora_target_modules, lora_rank, lora_checkpoint,
+            preset_lora_path, preset_lora_model,
+            task=task,
+        )
+
+        # Store other configs
+        self.use_gradient_checkpointing = use_gradient_checkpointing
+        self.use_gradient_checkpointing_offload = use_gradient_checkpointing_offload
+        self.extra_inputs = extra_inputs.split(",") if extra_inputs is not None else []
+        self.fp8_models = fp8_models
+        self.task = task
+        self.zero_cond_t = zero_cond_t
+        self.max_timestep_boundary = max_timestep_boundary
+        self.min_timestep_boundary = min_timestep_boundary
+        self.task_to_loss = {
+            "sft:data_process": lambda pipe, *args: args,
+            "sft": lambda pipe, inputs_shared, inputs_posi, inputs_nega: FlowMatchSFTLoss(pipe, **inputs_shared, **inputs_posi),
+            "sft:train": lambda pipe, inputs_shared, inputs_posi, inputs_nega: FlowMatchSFTLoss(pipe, **inputs_shared, **inputs_posi),
+        }
+
+    def get_pipeline_inputs(self, data):
+        inputs_posi = {"prompt": data["prompt"]}
+        inputs_nega = {"negative_prompt": ""}
+        input_video = data["input_video"]
+        num_frames = len(input_video)
+        inputs_shared = {
+            "edit_video": input_video,
+            "input_video": data["video"],
+            "chunk_id": 0,
+            "num_frames": num_frames,
+            "height": input_video[0].size[1],
+            "width": input_video[0].size[0],
+            "tiled": False,
+            "tile_size": (30, 52),
+            "tile_stride": (15, 26),
+            "cfg_scale": 1,
+            "rand_device": self.pipe.device,
+            "use_gradient_checkpointing": self.use_gradient_checkpointing,
+            "use_gradient_checkpointing_offload": self.use_gradient_checkpointing_offload,
+            "zero_cond_t": self.zero_cond_t,
+            "max_timestep_boundary": self.max_timestep_boundary,
+            "min_timestep_boundary": self.min_timestep_boundary,
+        }
+        inputs_shared = self.parse_extra_inputs(data, self.extra_inputs, inputs_shared)
+        return inputs_shared, inputs_posi, inputs_nega
+
+    def forward(self, data, inputs=None):
+        if inputs is None: inputs = self.get_pipeline_inputs(data)
+        inputs = self.transfer_data_to_device(inputs, self.pipe.device, self.pipe.torch_dtype)
+        for unit in self.pipe.units:
+            inputs = self.pipe.unit_runner(unit, self.pipe, *inputs)
+        loss = self.task_to_loss[self.task](self.pipe, *inputs)
+        return loss
+
+
+def qwen_video_edit_parser():
+    parser = argparse.ArgumentParser(description="Simple example of a training script.")
+    parser = add_general_config(parser)
+    parser = add_video_size_config(parser)
+    parser.add_argument("--tokenizer_path", type=str, default=None, help="Path to tokenizer.")
+    parser.add_argument("--processor_path", type=str, default=None, help="Path to the processor. If provided, the processor will be used for image editing.")
+    parser.add_argument("--zero_cond_t", default=False, action="store_true", help="A special parameter introduced by Qwen-Image-Edit-2511. Please enable it for this model.")
+    parser.add_argument("--max_timestep_boundary", type=float, default=1.0, help="Max timestep boundary.")
+    parser.add_argument("--min_timestep_boundary", type=float, default=0.0, help="Min timestep boundary.")
+    parser.add_argument("--initialize_model_on_cpu", default=False, action="store_true", help="Whether to initialize models on CPU.")
+    return parser
+
+
+if __name__ == "__main__":
+    parser = qwen_video_edit_parser()
+    args = parser.parse_args()
+    accelerator = accelerate.Accelerator(
+        gradient_accumulation_steps=args.gradient_accumulation_steps,
+        kwargs_handlers=[accelerate.DistributedDataParallelKwargs(find_unused_parameters=args.find_unused_parameters)],
+    )
+    dataset = UnifiedDataset(
+        base_path=args.dataset_base_path,
+        metadata_path=args.dataset_metadata_path,
+        repeat=args.dataset_repeat,
+        data_file_keys=args.data_file_keys.split(","),
+        main_data_operator=UnifiedDataset.default_video_operator(
+            base_path=args.dataset_base_path,
+            max_pixels=args.max_pixels,
+            height=args.height,
+            width=args.width,
+            height_division_factor=16,
+            width_division_factor=16,
+            num_frames=args.num_frames,
+            time_division_factor=4,
+            time_division_remainder=1,
+        ),
+    )
+    model = QwenVideoEditTrainingModule(
+        model_paths=args.model_paths,
+        model_id_with_origin_paths=args.model_id_with_origin_paths,
+        tokenizer_path=args.tokenizer_path,
+        processor_path=args.processor_path,
+        trainable_models=args.trainable_models,
+        lora_base_model=args.lora_base_model,
+        lora_target_modules=args.lora_target_modules,
+        lora_rank=args.lora_rank,
+        lora_checkpoint=args.lora_checkpoint,
+        preset_lora_path=args.preset_lora_path,
+        preset_lora_model=args.preset_lora_model,
+        use_gradient_checkpointing=args.use_gradient_checkpointing,
+        use_gradient_checkpointing_offload=args.use_gradient_checkpointing_offload,
+        extra_inputs=args.extra_inputs,
+        fp8_models=args.fp8_models,
+        offload_models=args.offload_models,
+        quant_options=args.quant_options,
+        resume_from_checkpoint=args.resume_from_checkpoint,
+        remove_prefix_in_ckpt=args.remove_prefix_in_ckpt,
+        task=args.task,
+        device="cpu" if (args.initialize_model_on_cpu or args.enable_model_cpu_offload) else accelerator.device,
+        zero_cond_t=args.zero_cond_t,
+        max_timestep_boundary=args.max_timestep_boundary,
+        min_timestep_boundary=args.min_timestep_boundary,
+    )
+    model_logger = ModelLogger(
+        args.output_path,
+        remove_prefix_in_ckpt=args.remove_prefix_in_ckpt,
+        enable_tensorboard_log=args.enable_tensorboard_log,
+        enable_swanlab_log=args.enable_swanlab_log,
+        swanlab_project=args.swanlab_project,
+        enable_wandb_log=args.enable_wandb_log,
+        wandb_project=args.wandb_project,
+        enable_csv_log=args.enable_csv_log,
+    )
+    launcher_map = {
+        "sft:data_process": launch_data_process_task,
+        "sft": launch_training_task,
+        "sft:train": launch_training_task,
+    }
+    launcher_map[args.task](accelerator, dataset, model, model_logger, args=args)

examples/qwen_video_edit/model_training/validate_full/Qwen-Video-Edit.py

@@ -0,0 +1,25 @@
+import torch
+
+from diffsynth import load_state_dict
+from diffsynth.core import ModelConfig
+from diffsynth.pipelines.qwen_video_edit import QwenVideoEditPipeline
+from diffsynth.utils.data import VideoData, save_video
+
+edit_video = VideoData("data/diffsynth_example_dataset/qwen_video_edit/Qwen-Video-Edit/source.mp4")
+prompts = [
+    "Transform the video into Japanese anime style",
+]
+pipe = QwenVideoEditPipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="yunpeng1998/Qwen-Video-Edit", origin_file_pattern="360P/step-30000.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors"),
+        ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="Wan2.1_VAE.pth"),
+    ],
+)
+state_dict = load_state_dict("models/train/Qwen-Video-Edit_full/epoch-1.safetensors")
+pipe.dit.load_state_dict(state_dict)
+
+video = pipe(edit_video=edit_video, prompts=prompts, height=640, width=384, num_frames=45, cfg_scale=4.0, num_inference_steps=40, seed=0)
+save_video(video, "video_Qwen-Video-Edit-full.mp4", fps=16)

examples/qwen_video_edit/model_training/validate_lora/Qwen-Video-Edit.py

@@ -0,0 +1,23 @@
+import torch
+
+from diffsynth.core import ModelConfig
+from diffsynth.pipelines.qwen_video_edit import QwenVideoEditPipeline
+from diffsynth.utils.data import VideoData, save_video
+
+edit_video = VideoData("data/diffsynth_example_dataset/qwen_video_edit/Qwen-Video-Edit/source.mp4")
+prompts = [
+    "Transform the video into Japanese anime style",
+]
+pipe = QwenVideoEditPipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="yunpeng1998/Qwen-Video-Edit", origin_file_pattern="360P/step-30000.safetensors"),
+        ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors"),
+        ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="Wan2.1_VAE.pth"),
+    ],
+)
+pipe.load_lora(pipe.dit, "models/train/Qwen-Video-Edit_lora/epoch-4.safetensors")
+
+video = pipe(edit_video=edit_video, prompts=prompts, height=640, width=384, num_frames=45, cfg_scale=4.0, num_inference_steps=40, seed=0)
+save_video(video, "video_Qwen-Video-Edit-lora.mp4", fps=16)

examples/stable_diffusion/model_inference/stable-diffusion-v1-5.py

@@ -0,0 +1,25 @@
+import torch
+from diffsynth.core import ModelConfig
+from diffsynth.pipelines.stable_diffusion import StableDiffusionPipeline
+
+pipe = StableDiffusionPipeline.from_pretrained(
+    torch_dtype=torch.float32,
+    model_configs=[
+        ModelConfig(model_id="AI-ModelScope/stable-diffusion-v1-5", origin_file_pattern="text_encoder/model.safetensors"),
+        ModelConfig(model_id="AI-ModelScope/stable-diffusion-v1-5", origin_file_pattern="unet/diffusion_pytorch_model.safetensors"),
+        ModelConfig(model_id="AI-ModelScope/stable-diffusion-v1-5", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"),
+    ],
+    tokenizer_config=ModelConfig(model_id="AI-ModelScope/stable-diffusion-v1-5", origin_file_pattern="tokenizer/"),
+)
+
+image = pipe(
+    prompt="a photo of an astronaut riding a horse on mars, high quality, detailed",
+    negative_prompt="blurry, low quality, deformed",
+    cfg_scale=7.5,
+    height=512,
+    width=512,
+    seed=42,
+    rand_device="cuda",
+    num_inference_steps=50,
+)
+image.save("image.jpg")

examples/stable_diffusion/model_inference_low_vram/stable-diffusion-v1-5.py

@@ -0,0 +1,36 @@
+import torch
+from diffsynth.core import ModelConfig
+from diffsynth.pipelines.stable_diffusion import StableDiffusionPipeline
+
+vram_config = {
+    "offload_dtype": torch.float32,
+    "offload_device": "cpu",
+    "onload_dtype": torch.float32,
+    "onload_device": "cpu",
+    "preparing_dtype": torch.float32,
+    "preparing_device": "cuda",
+    "computation_dtype": torch.float32,
+    "computation_device": "cuda",
+}
+pipe = StableDiffusionPipeline.from_pretrained(
+    torch_dtype=torch.float32,
+    model_configs=[
+        ModelConfig(model_id="AI-ModelScope/stable-diffusion-v1-5", origin_file_pattern="text_encoder/model.safetensors", **vram_config),
+        ModelConfig(model_id="AI-ModelScope/stable-diffusion-v1-5", origin_file_pattern="unet/diffusion_pytorch_model.safetensors", **vram_config),
+        ModelConfig(model_id="AI-ModelScope/stable-diffusion-v1-5", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config),
+    ],
+    tokenizer_config=ModelConfig(model_id="AI-ModelScope/stable-diffusion-v1-5", origin_file_pattern="tokenizer/"),
+    vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5,
+)
+
+image = pipe(
+    prompt="a photo of an astronaut riding a horse on mars, high quality, detailed",
+    negative_prompt="blurry, low quality, deformed",
+    cfg_scale=7.5,
+    height=512,
+    width=512,
+    seed=42,
+    rand_device="cuda",
+    num_inference_steps=50,
+)
+image.save("image.jpg")

examples/stable_diffusion/model_training/full/stable-diffusion-v1-5.sh

@@ -0,0 +1,15 @@
+modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "stable_diffusion/stable-diffusion-v1-5/*" --local_dir ./data/diffsynth_example_dataset
+
+accelerate launch examples/stable_diffusion/model_training/train.py \
+  --dataset_base_path data/diffsynth_example_dataset/stable_diffusion/stable-diffusion-v1-5 \
+  --dataset_metadata_path data/diffsynth_example_dataset/stable_diffusion/stable-diffusion-v1-5/metadata.csv \
+  --height 512 \
+  --width 512 \
+  --dataset_repeat 50 \
+  --model_id_with_origin_paths "AI-ModelScope/stable-diffusion-v1-5:text_encoder/model.safetensors,AI-ModelScope/stable-diffusion-v1-5:unet/diffusion_pytorch_model.safetensors,AI-ModelScope/stable-diffusion-v1-5:vae/diffusion_pytorch_model.safetensors" \
+  --learning_rate 1e-5 \
+  --num_epochs 2 \
+  --trainable_models "unet" \
+  --remove_prefix_in_ckpt "pipe.unet." \
+  --output_path "./models/train/stable-diffusion-v1-5_full" \
+  --use_gradient_checkpointing

examples/stable_diffusion/model_training/lora/stable-diffusion-v1-5.sh

@@ -0,0 +1,17 @@
+modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "stable_diffusion/stable-diffusion-v1-5/*" --local_dir ./data/diffsynth_example_dataset
+
+accelerate launch examples/stable_diffusion/model_training/train.py \
+  --dataset_base_path data/diffsynth_example_dataset/stable_diffusion/stable-diffusion-v1-5 \
+  --dataset_metadata_path data/diffsynth_example_dataset/stable_diffusion/stable-diffusion-v1-5/metadata.csv \
+  --height 512 \
+  --width 512 \
+  --dataset_repeat 50 \
+  --model_id_with_origin_paths "AI-ModelScope/stable-diffusion-v1-5:text_encoder/model.safetensors,AI-ModelScope/stable-diffusion-v1-5:unet/diffusion_pytorch_model.safetensors,AI-ModelScope/stable-diffusion-v1-5:vae/diffusion_pytorch_model.safetensors" \
+  --learning_rate 1e-4 \
+  --num_epochs 5 \
+  --remove_prefix_in_ckpt "pipe.unet." \
+  --output_path "./models/train/stable-diffusion-v1-5_lora" \
+  --lora_base_model "unet" \
+  --lora_target_modules "" \
+  --lora_rank 32 \
+  --use_gradient_checkpointing

examples/stable_diffusion/model_training/special/split_training/stable-diffusion-v1-5.sh

@@ -0,0 +1,38 @@
+modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "stable_diffusion/stable-diffusion-v1-5/*" --local_dir ./data/diffsynth_example_dataset
+
+# Stage 1: cache deterministic preprocessing outputs.
+accelerate launch examples/stable_diffusion/model_training/train.py \
+  --dataset_base_path data/diffsynth_example_dataset/stable_diffusion/stable-diffusion-v1-5 \
+  --dataset_metadata_path data/diffsynth_example_dataset/stable_diffusion/stable-diffusion-v1-5/metadata.csv \
+  --height 512 \
+  --width 512 \
+  --dataset_repeat 1 \
+  --model_id_with_origin_paths AI-ModelScope/stable-diffusion-v1-5:text_encoder/model.safetensors,AI-ModelScope/stable-diffusion-v1-5:unet/diffusion_pytorch_model.safetensors,AI-ModelScope/stable-diffusion-v1-5:vae/diffusion_pytorch_model.safetensors \
+  --learning_rate 1e-4 \
+  --num_epochs 5 \
+  --remove_prefix_in_ckpt pipe.unet. \
+  --output_path ./models/train/stable-diffusion-v1-5_split_cache \
+  --lora_base_model unet \
+  --lora_target_modules '' \
+  --lora_rank 32 \
+  --use_gradient_checkpointing \
+  --offload_models AI-ModelScope/stable-diffusion-v1-5:unet/diffusion_pytorch_model.safetensors \
+  --task sft:data_process
+
+# Stage 2: train LoRA from the cached dataset.
+accelerate launch examples/stable_diffusion/model_training/train.py \
+  --dataset_base_path ./models/train/stable-diffusion-v1-5_split_cache \
+  --height 512 \
+  --width 512 \
+  --dataset_repeat 50 \
+  --model_id_with_origin_paths AI-ModelScope/stable-diffusion-v1-5:text_encoder/model.safetensors,AI-ModelScope/stable-diffusion-v1-5:unet/diffusion_pytorch_model.safetensors,AI-ModelScope/stable-diffusion-v1-5:vae/diffusion_pytorch_model.safetensors \
+  --learning_rate 1e-4 \
+  --num_epochs 5 \
+  --remove_prefix_in_ckpt pipe.unet. \
+  --output_path ./models/train/stable-diffusion-v1-5_split \
+  --lora_base_model unet \
+  --lora_target_modules '' \
+  --lora_rank 32 \
+  --use_gradient_checkpointing \
+  --offload_models AI-ModelScope/stable-diffusion-v1-5:text_encoder/model.safetensors,AI-ModelScope/stable-diffusion-v1-5:vae/diffusion_pytorch_model.safetensors \
+  --task sft:train

examples/stable_diffusion/model_training/special/split_training/validate.py

@@ -0,0 +1,26 @@
+import torch
+from diffsynth.core import ModelConfig
+from diffsynth.pipelines.stable_diffusion import StableDiffusionPipeline
+
+pipe = StableDiffusionPipeline.from_pretrained(
+    torch_dtype=torch.float32,
+    model_configs=[
+        ModelConfig(model_id="AI-ModelScope/stable-diffusion-v1-5", origin_file_pattern="text_encoder/model.safetensors"),
+        ModelConfig(model_id="AI-ModelScope/stable-diffusion-v1-5", origin_file_pattern="unet/diffusion_pytorch_model.safetensors"),
+        ModelConfig(model_id="AI-ModelScope/stable-diffusion-v1-5", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"),
+    ],
+    tokenizer_config=ModelConfig(model_id="AI-ModelScope/stable-diffusion-v1-5", origin_file_pattern="tokenizer/"),
+)
+pipe.load_lora(pipe.unet, './models/train/stable-diffusion-v1-5_split/epoch-4.safetensors')
+
+image = pipe(
+    prompt="a dog",
+    negative_prompt="blurry, low quality, deformed",
+    cfg_scale=7.5,
+    height=512,
+    width=512,
+    seed=42,
+    rand_device="cuda",
+    num_inference_steps=50,
+)
+image.save('split_training_stable-diffusion-v1-5.jpg')

examples/stable_diffusion/model_training/train.py

@@ -0,0 +1,154 @@
+import torch, os, argparse, accelerate
+from diffsynth.core import UnifiedDataset
+from diffsynth.pipelines.stable_diffusion import StableDiffusionPipeline, ModelConfig
+from diffsynth.diffusion import *
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+
+class StableDiffusionTrainingModule(DiffusionTrainingModule):
+    def __init__(
+        self,
+        model_paths=None, model_id_with_origin_paths=None,
+        tokenizer_path=None,
+        trainable_models=None,
+        lora_base_model=None, lora_target_modules="", lora_rank=32, lora_checkpoint=None,
+        preset_lora_path=None, preset_lora_model=None,
+        use_gradient_checkpointing=True,
+        use_gradient_checkpointing_offload=False,
+        extra_inputs=None,
+        fp8_models=None,
+        offload_models=None,
+        quant_options=None,
+        resume_from_checkpoint=None, remove_prefix_in_ckpt=None,
+        device="cpu",
+        task="sft",
+    ):
+        super().__init__()
+        # Load models
+        model_configs = self.parse_model_configs(model_paths, model_id_with_origin_paths, fp8_models=fp8_models, offload_models=offload_models, quant_options=quant_options, device=device)
+        tokenizer_config = self.parse_path_or_model_id(tokenizer_path, ModelConfig(model_id="AI-ModelScope/stable-diffusion-v1-5", origin_file_pattern="tokenizer/"))
+        self.pipe = StableDiffusionPipeline.from_pretrained(torch_dtype=torch.float32, device=device, model_configs=model_configs, tokenizer_config=tokenizer_config)
+        self.pipe = self.split_pipeline_units(task, self.pipe, trainable_models, lora_base_model)
+        self.resume_from_checkpoint(resume_from_checkpoint, remove_prefix_in_ckpt)
+
+        # Training mode
+        self.switch_pipe_to_training_mode(
+            self.pipe, trainable_models,
+            lora_base_model, lora_target_modules, lora_rank, lora_checkpoint,
+            preset_lora_path, preset_lora_model,
+            task=task,
+        )
+        
+        # Other configs
+        self.use_gradient_checkpointing = use_gradient_checkpointing
+        self.use_gradient_checkpointing_offload = use_gradient_checkpointing_offload
+        self.extra_inputs = extra_inputs.split(",") if extra_inputs is not None else []
+        self.fp8_models = fp8_models
+        self.task = task
+        self.task_to_loss = {
+            "sft:data_process": lambda pipe, *args: args,
+            "direct_distill:data_process": lambda pipe, *args: args,
+            "sft": lambda pipe, inputs_shared, inputs_posi, inputs_nega: FlowMatchSFTLoss(pipe, **inputs_shared, **inputs_posi),
+            "sft:train": lambda pipe, inputs_shared, inputs_posi, inputs_nega: FlowMatchSFTLoss(pipe, **inputs_shared, **inputs_posi),
+            "direct_distill": lambda pipe, inputs_shared, inputs_posi, inputs_nega: DirectDistillLoss(pipe, **inputs_shared, **inputs_posi),
+            "direct_distill:train": lambda pipe, inputs_shared, inputs_posi, inputs_nega: DirectDistillLoss(pipe, **inputs_shared, **inputs_posi),
+        }
+        
+    def get_pipeline_inputs(self, data):
+        inputs_posi = {"prompt": data["prompt"]}
+        inputs_nega = {"negative_prompt": ""}
+        inputs_shared = {
+            # Assume you are using this pipeline for inference,
+            # please fill in the input parameters.
+            "input_image": data["image"],
+            "height": data["image"].size[1],
+            "width": data["image"].size[0],
+            # Please do not modify the following parameters
+            # unless you clearly know what this will cause.
+            "cfg_scale": 1,
+            "rand_device": self.pipe.device,
+            "use_gradient_checkpointing": self.use_gradient_checkpointing,
+            "use_gradient_checkpointing_offload": self.use_gradient_checkpointing_offload,
+        }
+        inputs_shared = self.parse_extra_inputs(data, self.extra_inputs, inputs_shared)
+        return inputs_shared, inputs_posi, inputs_nega
+    
+    def forward(self, data, inputs=None):
+        if inputs is None: inputs = self.get_pipeline_inputs(data)
+        inputs = self.transfer_data_to_device(inputs, self.pipe.device, self.pipe.torch_dtype)
+        for unit in self.pipe.units:
+            inputs = self.pipe.unit_runner(unit, self.pipe, *inputs)
+        loss = self.task_to_loss[self.task](self.pipe, *inputs)
+        return loss
+
+
+def parser():
+    parser = argparse.ArgumentParser(description="Simple example of a training script.")
+    parser = add_general_config(parser)
+    parser = add_image_size_config(parser)
+    parser.add_argument("--tokenizer_path", type=str, default=None, help="Path to tokenizer.")
+    return parser
+
+
+if __name__ == "__main__":
+    parser = parser()
+    args = parser.parse_args()
+    accelerator = accelerate.Accelerator(
+        gradient_accumulation_steps=args.gradient_accumulation_steps,
+        kwargs_handlers=[accelerate.DistributedDataParallelKwargs(find_unused_parameters=args.find_unused_parameters)],
+    )
+    dataset = UnifiedDataset(
+        base_path=args.dataset_base_path,
+        metadata_path=args.dataset_metadata_path,
+        repeat=args.dataset_repeat,
+        data_file_keys=args.data_file_keys.split(","),
+        main_data_operator=UnifiedDataset.default_image_operator(
+            base_path=args.dataset_base_path,
+            max_pixels=args.max_pixels,
+            height=args.height,
+            width=args.width,
+            height_division_factor=32,
+            width_division_factor=32,
+        )
+    )
+    model = StableDiffusionTrainingModule(
+        model_paths=args.model_paths,
+        model_id_with_origin_paths=args.model_id_with_origin_paths,
+        tokenizer_path=args.tokenizer_path,
+        trainable_models=args.trainable_models,
+        lora_base_model=args.lora_base_model,
+        lora_target_modules=args.lora_target_modules,
+        lora_rank=args.lora_rank,
+        lora_checkpoint=args.lora_checkpoint,
+        preset_lora_path=args.preset_lora_path,
+        preset_lora_model=args.preset_lora_model,
+        use_gradient_checkpointing=args.use_gradient_checkpointing,
+        use_gradient_checkpointing_offload=args.use_gradient_checkpointing_offload,
+        extra_inputs=args.extra_inputs,
+        fp8_models=args.fp8_models,
+        offload_models=args.offload_models,
+        quant_options=args.quant_options,
+        resume_from_checkpoint=args.resume_from_checkpoint,
+        remove_prefix_in_ckpt=args.remove_prefix_in_ckpt,
+        task=args.task,
+        device="cpu" if args.enable_model_cpu_offload else accelerator.device,
+    )
+    model_logger = ModelLogger(
+        args.output_path,
+        remove_prefix_in_ckpt=args.remove_prefix_in_ckpt,
+        enable_tensorboard_log=args.enable_tensorboard_log,
+        enable_swanlab_log=args.enable_swanlab_log,
+        swanlab_project=args.swanlab_project,
+        enable_wandb_log=args.enable_wandb_log,
+        wandb_project=args.wandb_project,
+        enable_csv_log=args.enable_csv_log,
+    )
+    launcher_map = {
+        "sft:data_process": launch_data_process_task,
+        "direct_distill:data_process": launch_data_process_task,
+        "sft": launch_training_task,
+        "sft:train": launch_training_task,
+        "direct_distill": launch_training_task,
+        "direct_distill:train": launch_training_task,
+    }
+    launcher_map[args.task](accelerator, dataset, model, model_logger, args=args)

examples/stable_diffusion/model_training/validate_full/stable-diffusion-v1-5.py

@@ -0,0 +1,27 @@
+from diffsynth.pipelines.stable_diffusion import StableDiffusionPipeline, ModelConfig
+from diffsynth.core import load_state_dict
+import torch
+
+pipe = StableDiffusionPipeline.from_pretrained(
+    torch_dtype=torch.float32,
+    model_configs=[
+        ModelConfig(model_id="AI-ModelScope/stable-diffusion-v1-5", origin_file_pattern="text_encoder/model.safetensors"),
+        ModelConfig(model_id="AI-ModelScope/stable-diffusion-v1-5", origin_file_pattern="unet/diffusion_pytorch_model.safetensors"),
+        ModelConfig(model_id="AI-ModelScope/stable-diffusion-v1-5", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"),
+    ],
+    tokenizer_config=ModelConfig(model_id="AI-ModelScope/stable-diffusion-v1-5", origin_file_pattern="tokenizer/"),
+)
+state_dict = load_state_dict("./models/train/stable-diffusion-v1-5_full/epoch-1.safetensors", torch_dtype=torch.float32)
+pipe.unet.load_state_dict(state_dict)
+
+image = pipe(
+    prompt="a dog",
+    negative_prompt="blurry, low quality, deformed",
+    cfg_scale=7.5,
+    height=512,
+    width=512,
+    seed=42,
+    rand_device="cuda",
+    num_inference_steps=50,
+)
+image.save("image_stable-diffusion-v1-5_full.jpg")

examples/stable_diffusion/model_training/validate_lora/stable-diffusion-v1-5.py

@@ -0,0 +1,26 @@
+import torch
+from diffsynth.core import ModelConfig
+from diffsynth.pipelines.stable_diffusion import StableDiffusionPipeline
+
+pipe = StableDiffusionPipeline.from_pretrained(
+    torch_dtype=torch.float32,
+    model_configs=[
+        ModelConfig(model_id="AI-ModelScope/stable-diffusion-v1-5", origin_file_pattern="text_encoder/model.safetensors"),
+        ModelConfig(model_id="AI-ModelScope/stable-diffusion-v1-5", origin_file_pattern="unet/diffusion_pytorch_model.safetensors"),
+        ModelConfig(model_id="AI-ModelScope/stable-diffusion-v1-5", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"),
+    ],
+    tokenizer_config=ModelConfig(model_id="AI-ModelScope/stable-diffusion-v1-5", origin_file_pattern="tokenizer/"),
+)
+pipe.load_lora(pipe.unet, "models/train/stable-diffusion-v1-5_lora/epoch-4.safetensors")
+
+image = pipe(
+    prompt="a dog",
+    negative_prompt="blurry, low quality, deformed",
+    cfg_scale=7.5,
+    height=512,
+    width=512,
+    seed=42,
+    rand_device="cuda",
+    num_inference_steps=50,
+)
+image.save("image_stable-diffusion-v1-5.jpg")

examples/stable_diffusion_xl/model_inference/stable-diffusion-xl-base-1.0.py

@@ -0,0 +1,26 @@
+import torch
+from diffsynth.core import ModelConfig
+from diffsynth.pipelines.stable_diffusion_xl import StableDiffusionXLPipeline
+
+pipe = StableDiffusionXLPipeline.from_pretrained(
+    torch_dtype=torch.float32,
+    model_configs=[
+        ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="text_encoder/model.safetensors"),
+        ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="text_encoder_2/model.safetensors"),
+        ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="unet/diffusion_pytorch_model.safetensors"),
+        ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"),
+    ],
+    tokenizer_config=ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="tokenizer/"),
+    tokenizer_2_config=ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="tokenizer_2/"),
+)
+
+image = pipe(
+    prompt="a photo of an astronaut riding a horse on mars",
+    negative_prompt="",
+    cfg_scale=5.0,
+    height=1024,
+    width=1024,
+    seed=42,
+    num_inference_steps=50,
+)
+image.save("image.jpg")

examples/stable_diffusion_xl/model_inference_low_vram/stable-diffusion-xl-base-1.0.py

@@ -0,0 +1,37 @@
+import torch
+from diffsynth.core import ModelConfig
+from diffsynth.pipelines.stable_diffusion_xl import StableDiffusionXLPipeline
+
+vram_config = {
+    "offload_dtype": torch.float32,
+    "offload_device": "cpu",
+    "onload_dtype": torch.float32,
+    "onload_device": "cpu",
+    "preparing_dtype": torch.float32,
+    "preparing_device": "cuda",
+    "computation_dtype": torch.float32,
+    "computation_device": "cuda",
+}
+pipe = StableDiffusionXLPipeline.from_pretrained(
+    torch_dtype=torch.float32,
+    model_configs=[
+        ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="text_encoder/model.safetensors", **vram_config),
+        ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="text_encoder_2/model.safetensors", **vram_config),
+        ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="unet/diffusion_pytorch_model.safetensors", **vram_config),
+        ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config),
+    ],
+    tokenizer_config=ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="tokenizer/"),
+    tokenizer_2_config=ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="tokenizer_2/"),
+    vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5,
+)
+
+image = pipe(
+    prompt="a photo of an astronaut riding a horse on mars",
+    negative_prompt="",
+    cfg_scale=5.0,
+    height=1024,
+    width=1024,
+    seed=42,
+    num_inference_steps=50,
+)
+image.save("image.jpg")

examples/stable_diffusion_xl/model_training/full/stable-diffusion-xl-base-1.0.sh

@@ -0,0 +1,15 @@
+modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "stable_diffusion_xl/stable-diffusion-xl-base-1.0/*" --local_dir ./data/diffsynth_example_dataset
+
+accelerate launch examples/stable_diffusion_xl/model_training/train.py \
+  --dataset_base_path data/diffsynth_example_dataset/stable_diffusion_xl/stable-diffusion-xl-base-1.0 \
+  --dataset_metadata_path data/diffsynth_example_dataset/stable_diffusion_xl/stable-diffusion-xl-base-1.0/metadata.csv \
+  --height 1024 \
+  --width 1024 \
+  --dataset_repeat 10 \
+  --model_id_with_origin_paths "stabilityai/stable-diffusion-xl-base-1.0:text_encoder/model.safetensors,stabilityai/stable-diffusion-xl-base-1.0:text_encoder_2/model.safetensors,stabilityai/stable-diffusion-xl-base-1.0:unet/diffusion_pytorch_model.safetensors,stabilityai/stable-diffusion-xl-base-1.0:vae/diffusion_pytorch_model.safetensors" \
+  --learning_rate 1e-5 \
+  --num_epochs 2 \
+  --trainable_models "unet" \
+  --remove_prefix_in_ckpt "pipe.unet." \
+  --output_path "./models/train/stable-diffusion-xl-base-1.0_full" \
+  --use_gradient_checkpointing

examples/stable_diffusion_xl/model_training/lora/stable-diffusion-xl-base-1.0.sh

@@ -0,0 +1,18 @@
+modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "stable_diffusion_xl/stable-diffusion-xl-base-1.0/*" --local_dir ./data/diffsynth_example_dataset
+
+accelerate launch examples/stable_diffusion_xl/model_training/train.py \
+  --dataset_base_path data/diffsynth_example_dataset/stable_diffusion_xl/stable-diffusion-xl-base-1.0 \
+  --dataset_metadata_path data/diffsynth_example_dataset/stable_diffusion_xl/stable-diffusion-xl-base-1.0/metadata.csv \
+  --height 1024 \
+  --width 1024 \
+  --dataset_repeat 10 \
+  --model_id_with_origin_paths "stabilityai/stable-diffusion-xl-base-1.0:text_encoder/model.safetensors,stabilityai/stable-diffusion-xl-base-1.0:text_encoder_2/model.safetensors,stabilityai/stable-diffusion-xl-base-1.0:unet/diffusion_pytorch_model.safetensors,stabilityai/stable-diffusion-xl-base-1.0:vae/diffusion_pytorch_model.safetensors" \
+  --learning_rate 1e-4 \
+  --num_epochs 5 \
+  --remove_prefix_in_ckpt "pipe.unet." \
+  --output_path "./models/train/stable-diffusion-xl-base-1.0_lora" \
+  --lora_base_model "unet" \
+  --lora_target_modules "mid_block.attentions.0.proj_in,mid_block.attentions.0.proj_out,down_blocks.1.attentions.0.proj_in,down_blocks.1.attentions.0.proj_out,down_blocks.1.attentions.0.transformer_blocks.0.attn1.to_k,down_blocks.1.attentions.0.transformer_blocks.0.attn1.to_out.0,down_blocks.1.attentions.0.transformer_blocks.0.attn1.to_q,down_blocks.1.attentions.0.transformer_blocks.0.attn1.to_v,down_blocks.1.attentions.0.transformer_blocks.0.attn2.to_k,down_blocks.1.attentions.0.transformer_blocks.0.attn2.to_out.0,down_blocks.1.attentions.0.transformer_blocks.0.attn2.to_q,down_blocks.1.attentions.0.transformer_blocks.0.attn2.to_v,down_blocks.1.attentions.0.transformer_blocks.0.ff.net.0.proj,down_blocks.1.attentions.0.transformer_blocks.0.ff.net.2,down_blocks.1.attentions.0.transformer_blocks.1.attn1.to_k,down_blocks.1.attentions.0.transformer_blocks.1.attn1.to_out.0,down_blocks.1.attentions.0.transformer_blocks.1.attn1.to_q,down_blocks.1.attentions.0.transformer_blocks.1.attn1.to_v,down_blocks.1.attentions.0.transformer_blocks.1.attn2.to_k,down_blocks.1.attentions.0.transformer_blocks.1.attn2.to_out.0,down_blocks.1.attentions.0.transformer_blocks.1.attn2.to_q,down_blocks.1.attentions.0.transformer_blocks.1.attn2.to_v,down_blocks.1.attentions.0.transformer_blocks.1.ff.net.0.proj,down_blocks.1.attentions.0.transformer_blocks.1.ff.net.2,down_blocks.1.attentions.1.proj_in,down_blocks.1.attentions.1.proj_out,down_blocks.1.attentions.1.transformer_blocks.0.attn1.to_k,down_blocks.1.attentions.1.transformer_blocks.0.attn1.to_out.0,down_blocks.1.attentions.1.transformer_blocks.0.attn1.to_q,down_blocks.1.attentions.1.transformer_blocks.0.attn1.to_v,down_blocks.1.attentions.1.transformer_blocks.0.attn2.to_k,down_blocks.1.attentions.1.transformer_blocks.0.attn2.to_out.0,down_blocks.1.attentions.1.transformer_blocks.0.attn2.to_q,down_blocks.1.attentions.1.transformer_blocks.0.attn2.to_v,down_blocks.1.attentions.1.transformer_blocks.0.ff.net.0.proj,down_blocks.1.attentions.1.transformer_blocks.0.ff.net.2,down_blocks.1.attentions.1.transformer_blocks.1.attn1.to_k,down_blocks.1.attentions.1.transformer_blocks.1.attn1.to_out.0,down_blocks.1.attentions.1.transformer_blocks.1.attn1.to_q,down_blocks.1.attentions.1.transformer_blocks.1.attn1.to_v,down_blocks.1.attentions.1.transformer_blocks.1.attn2.to_k,down_blocks.1.attentions.1.transformer_blocks.1.attn2.to_out.0,down_blocks.1.attentions.1.transformer_blocks.1.attn2.to_q,down_blocks.1.attentions.1.transformer_blocks.1.attn2.to_v,down_blocks.1.attentions.1.transformer_blocks.1.ff.net.0.proj,down_blocks.1.attentions.1.transformer_blocks.1.ff.net.2,down_blocks.2.attentions.0.proj_in,down_blocks.2.attentions.0.proj_out,down_blocks.2.attentions.0.transformer_blocks.0.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.0.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.0.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.0.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.0.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.0.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.0.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.0.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.0.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.0.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.1.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.1.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.1.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.1.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.1.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.1.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.1.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.1.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.1.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.1.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.2.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.2.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.2.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.2.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.2.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.2.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.2.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.2.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.2.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.2.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.3.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.3.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.3.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.3.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.3.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.3.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.3.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.3.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.3.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.3.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.4.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.4.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.4.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.4.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.4.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.4.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.4.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.4.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.4.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.4.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.5.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.5.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.5.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.5.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.5.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.5.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.5.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.5.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.5.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.5.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.6.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.6.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.6.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.6.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.6.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.6.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.6.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.6.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.6.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.6.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.7.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.7.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.7.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.7.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.7.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.7.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.7.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.7.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.7.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.7.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.8.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.8.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.8.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.8.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.8.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.8.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.8.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.8.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.8.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.8.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.9.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.9.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.9.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.9.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.9.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.9.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.9.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.9.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.9.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.9.ff.net.2,down_blocks.2.attentions.1.proj_in,down_blocks.2.attentions.1.proj_out,down_blocks.2.attentions.1.transformer_blocks.0.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.0.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.0.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.0.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.0.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.0.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.0.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.0.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.0.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.0.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.1.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.1.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.1.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.1.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.1.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.1.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.1.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.1.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.1.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.1.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.2.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.2.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.2.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.2.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.2.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.2.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.2.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.2.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.2.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.2.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.3.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.3.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.3.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.3.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.3.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.3.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.3.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.3.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.3.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.3.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.4.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.4.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.4.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.4.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.4.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.4.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.4.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.4.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.4.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.4.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.5.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.5.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.5.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.5.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.5.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.5.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.5.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.5.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.5.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.5.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.6.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.6.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.6.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.6.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.6.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.6.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.6.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.6.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.6.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.6.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.7.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.7.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.7.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.7.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.7.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.7.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.7.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.7.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.7.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.7.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.8.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.8.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.8.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.8.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.8.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.8.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.8.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.8.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.8.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.8.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.9.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.9.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.9.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.9.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.9.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.9.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.9.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.9.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.9.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.9.ff.net.2,mid_block.attentions.0.transformer_blocks.0.attn1.to_k,mid_block.attentions.0.transformer_blocks.0.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.0.attn1.to_q,mid_block.attentions.0.transformer_blocks.0.attn1.to_v,mid_block.attentions.0.transformer_blocks.0.attn2.to_k,mid_block.attentions.0.transformer_blocks.0.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.0.attn2.to_q,mid_block.attentions.0.transformer_blocks.0.attn2.to_v,mid_block.attentions.0.transformer_blocks.0.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.0.ff.net.2,mid_block.attentions.0.transformer_blocks.1.attn1.to_k,mid_block.attentions.0.transformer_blocks.1.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.1.attn1.to_q,mid_block.attentions.0.transformer_blocks.1.attn1.to_v,mid_block.attentions.0.transformer_blocks.1.attn2.to_k,mid_block.attentions.0.transformer_blocks.1.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.1.attn2.to_q,mid_block.attentions.0.transformer_blocks.1.attn2.to_v,mid_block.attentions.0.transformer_blocks.1.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.1.ff.net.2,mid_block.attentions.0.transformer_blocks.2.attn1.to_k,mid_block.attentions.0.transformer_blocks.2.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.2.attn1.to_q,mid_block.attentions.0.transformer_blocks.2.attn1.to_v,mid_block.attentions.0.transformer_blocks.2.attn2.to_k,mid_block.attentions.0.transformer_blocks.2.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.2.attn2.to_q,mid_block.attentions.0.transformer_blocks.2.attn2.to_v,mid_block.attentions.0.transformer_blocks.2.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.2.ff.net.2,mid_block.attentions.0.transformer_blocks.3.attn1.to_k,mid_block.attentions.0.transformer_blocks.3.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.3.attn1.to_q,mid_block.attentions.0.transformer_blocks.3.attn1.to_v,mid_block.attentions.0.transformer_blocks.3.attn2.to_k,mid_block.attentions.0.transformer_blocks.3.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.3.attn2.to_q,mid_block.attentions.0.transformer_blocks.3.attn2.to_v,mid_block.attentions.0.transformer_blocks.3.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.3.ff.net.2,mid_block.attentions.0.transformer_blocks.4.attn1.to_k,mid_block.attentions.0.transformer_blocks.4.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.4.attn1.to_q,mid_block.attentions.0.transformer_blocks.4.attn1.to_v,mid_block.attentions.0.transformer_blocks.4.attn2.to_k,mid_block.attentions.0.transformer_blocks.4.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.4.attn2.to_q,mid_block.attentions.0.transformer_blocks.4.attn2.to_v,mid_block.attentions.0.transformer_blocks.4.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.4.ff.net.2,mid_block.attentions.0.transformer_blocks.5.attn1.to_k,mid_block.attentions.0.transformer_blocks.5.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.5.attn1.to_q,mid_block.attentions.0.transformer_blocks.5.attn1.to_v,mid_block.attentions.0.transformer_blocks.5.attn2.to_k,mid_block.attentions.0.transformer_blocks.5.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.5.attn2.to_q,mid_block.attentions.0.transformer_blocks.5.attn2.to_v,mid_block.attentions.0.transformer_blocks.5.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.5.ff.net.2,mid_block.attentions.0.transformer_blocks.6.attn1.to_k,mid_block.attentions.0.transformer_blocks.6.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.6.attn1.to_q,mid_block.attentions.0.transformer_blocks.6.attn1.to_v,mid_block.attentions.0.transformer_blocks.6.attn2.to_k,mid_block.attentions.0.transformer_blocks.6.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.6.attn2.to_q,mid_block.attentions.0.transformer_blocks.6.attn2.to_v,mid_block.attentions.0.transformer_blocks.6.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.6.ff.net.2,mid_block.attentions.0.transformer_blocks.7.attn1.to_k,mid_block.attentions.0.transformer_blocks.7.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.7.attn1.to_q,mid_block.attentions.0.transformer_blocks.7.attn1.to_v,mid_block.attentions.0.transformer_blocks.7.attn2.to_k,mid_block.attentions.0.transformer_blocks.7.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.7.attn2.to_q,mid_block.attentions.0.transformer_blocks.7.attn2.to_v,mid_block.attentions.0.transformer_blocks.7.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.7.ff.net.2,mid_block.attentions.0.transformer_blocks.8.attn1.to_k,mid_block.attentions.0.transformer_blocks.8.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.8.attn1.to_q,mid_block.attentions.0.transformer_blocks.8.attn1.to_v,mid_block.attentions.0.transformer_blocks.8.attn2.to_k,mid_block.attentions.0.transformer_blocks.8.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.8.attn2.to_q,mid_block.attentions.0.transformer_blocks.8.attn2.to_v,mid_block.attentions.0.transformer_blocks.8.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.8.ff.net.2,mid_block.attentions.0.transformer_blocks.9.attn1.to_k,mid_block.attentions.0.transformer_blocks.9.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.9.attn1.to_q,mid_block.attentions.0.transformer_blocks.9.attn1.to_v,mid_block.attentions.0.transformer_blocks.9.attn2.to_k,mid_block.attentions.0.transformer_blocks.9.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.9.attn2.to_q,mid_block.attentions.0.transformer_blocks.9.attn2.to_v,mid_block.attentions.0.transformer_blocks.9.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.9.ff.net.2,up_blocks.0.attentions.0.proj_in,up_blocks.0.attentions.0.proj_out,up_blocks.0.attentions.0.transformer_blocks.0.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.0.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.0.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.0.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.0.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.0.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.0.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.0.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.0.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.0.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.1.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.1.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.1.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.1.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.1.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.1.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.1.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.1.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.1.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.1.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.2.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.2.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.2.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.2.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.2.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.2.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.2.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.2.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.2.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.2.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.3.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.3.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.3.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.3.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.3.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.3.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.3.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.3.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.3.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.3.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.4.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.4.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.4.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.4.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.4.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.4.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.4.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.4.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.4.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.4.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.5.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.5.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.5.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.5.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.5.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.5.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.5.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.5.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.5.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.5.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.6.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.6.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.6.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.6.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.6.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.6.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.6.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.6.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.6.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.6.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.7.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.7.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.7.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.7.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.7.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.7.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.7.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.7.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.7.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.7.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.8.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.8.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.8.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.8.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.8.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.8.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.8.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.8.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.8.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.8.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.9.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.9.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.9.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.9.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.9.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.9.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.9.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.9.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.9.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.9.ff.net.2,up_blocks.0.attentions.1.proj_in,up_blocks.0.attentions.1.proj_out,up_blocks.0.attentions.1.transformer_blocks.0.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.0.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.0.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.0.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.0.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.0.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.0.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.0.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.0.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.0.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.1.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.1.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.1.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.1.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.1.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.1.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.1.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.1.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.1.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.1.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.2.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.2.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.2.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.2.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.2.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.2.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.2.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.2.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.2.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.2.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.3.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.3.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.3.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.3.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.3.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.3.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.3.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.3.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.3.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.3.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.4.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.4.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.4.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.4.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.4.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.4.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.4.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.4.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.4.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.4.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.5.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.5.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.5.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.5.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.5.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.5.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.5.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.5.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.5.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.5.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.6.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.6.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.6.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.6.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.6.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.6.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.6.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.6.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.6.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.6.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.7.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.7.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.7.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.7.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.7.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.7.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.7.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.7.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.7.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.7.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.8.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.8.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.8.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.8.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.8.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.8.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.8.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.8.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.8.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.8.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.9.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.9.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.9.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.9.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.9.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.9.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.9.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.9.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.9.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.9.ff.net.2,up_blocks.0.attentions.2.proj_in,up_blocks.0.attentions.2.proj_out,up_blocks.0.attentions.2.transformer_blocks.0.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.0.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.0.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.0.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.0.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.0.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.0.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.0.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.0.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.0.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.1.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.1.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.1.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.1.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.1.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.1.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.1.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.1.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.1.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.1.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.2.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.2.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.2.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.2.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.2.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.2.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.2.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.2.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.2.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.2.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.3.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.3.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.3.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.3.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.3.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.3.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.3.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.3.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.3.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.3.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.4.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.4.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.4.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.4.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.4.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.4.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.4.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.4.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.4.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.4.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.5.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.5.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.5.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.5.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.5.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.5.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.5.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.5.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.5.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.5.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.6.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.6.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.6.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.6.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.6.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.6.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.6.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.6.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.6.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.6.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.7.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.7.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.7.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.7.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.7.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.7.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.7.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.7.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.7.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.7.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.8.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.8.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.8.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.8.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.8.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.8.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.8.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.8.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.8.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.8.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.9.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.9.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.9.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.9.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.9.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.9.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.9.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.9.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.9.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.9.ff.net.2,up_blocks.1.attentions.0.proj_in,up_blocks.1.attentions.0.proj_out,up_blocks.1.attentions.0.transformer_blocks.0.attn1.to_k,up_blocks.1.attentions.0.transformer_blocks.0.attn1.to_out.0,up_blocks.1.attentions.0.transformer_blocks.0.attn1.to_q,up_blocks.1.attentions.0.transformer_blocks.0.attn1.to_v,up_blocks.1.attentions.0.transformer_blocks.0.attn2.to_k,up_blocks.1.attentions.0.transformer_blocks.0.attn2.to_out.0,up_blocks.1.attentions.0.transformer_blocks.0.attn2.to_q,up_blocks.1.attentions.0.transformer_blocks.0.attn2.to_v,up_blocks.1.attentions.0.transformer_blocks.0.ff.net.0.proj,up_blocks.1.attentions.0.transformer_blocks.0.ff.net.2,up_blocks.1.attentions.0.transformer_blocks.1.attn1.to_k,up_blocks.1.attentions.0.transformer_blocks.1.attn1.to_out.0,up_blocks.1.attentions.0.transformer_blocks.1.attn1.to_q,up_blocks.1.attentions.0.transformer_blocks.1.attn1.to_v,up_blocks.1.attentions.0.transformer_blocks.1.attn2.to_k,up_blocks.1.attentions.0.transformer_blocks.1.attn2.to_out.0,up_blocks.1.attentions.0.transformer_blocks.1.attn2.to_q,up_blocks.1.attentions.0.transformer_blocks.1.attn2.to_v,up_blocks.1.attentions.0.transformer_blocks.1.ff.net.0.proj,up_blocks.1.attentions.0.transformer_blocks.1.ff.net.2,up_blocks.1.attentions.1.proj_in,up_blocks.1.attentions.1.proj_out,up_blocks.1.attentions.1.transformer_blocks.0.attn1.to_k,up_blocks.1.attentions.1.transformer_blocks.0.attn1.to_out.0,up_blocks.1.attentions.1.transformer_blocks.0.attn1.to_q,up_blocks.1.attentions.1.transformer_blocks.0.attn1.to_v,up_blocks.1.attentions.1.transformer_blocks.0.attn2.to_k,up_blocks.1.attentions.1.transformer_blocks.0.attn2.to_out.0,up_blocks.1.attentions.1.transformer_blocks.0.attn2.to_q,up_blocks.1.attentions.1.transformer_blocks.0.attn2.to_v,up_blocks.1.attentions.1.transformer_blocks.0.ff.net.0.proj,up_blocks.1.attentions.1.transformer_blocks.0.ff.net.2,up_blocks.1.attentions.1.transformer_blocks.1.attn1.to_k,up_blocks.1.attentions.1.transformer_blocks.1.attn1.to_out.0,up_blocks.1.attentions.1.transformer_blocks.1.attn1.to_q,up_blocks.1.attentions.1.transformer_blocks.1.attn1.to_v,up_blocks.1.attentions.1.transformer_blocks.1.attn2.to_k,up_blocks.1.attentions.1.transformer_blocks.1.attn2.to_out.0,up_blocks.1.attentions.1.transformer_blocks.1.attn2.to_q,up_blocks.1.attentions.1.transformer_blocks.1.attn2.to_v,up_blocks.1.attentions.1.transformer_blocks.1.ff.net.0.proj,up_blocks.1.attentions.1.transformer_blocks.1.ff.net.2,up_blocks.1.attentions.2.proj_in,up_blocks.1.attentions.2.proj_out,up_blocks.1.attentions.2.transformer_blocks.0.attn1.to_k,up_blocks.1.attentions.2.transformer_blocks.0.attn1.to_out.0,up_blocks.1.attentions.2.transformer_blocks.0.attn1.to_q,up_blocks.1.attentions.2.transformer_blocks.0.attn1.to_v,up_blocks.1.attentions.2.transformer_blocks.0.attn2.to_k,up_blocks.1.attentions.2.transformer_blocks.0.attn2.to_out.0,up_blocks.1.attentions.2.transformer_blocks.0.attn2.to_q,up_blocks.1.attentions.2.transformer_blocks.0.attn2.to_v,up_blocks.1.attentions.2.transformer_blocks.0.ff.net.0.proj,up_blocks.1.attentions.2.transformer_blocks.0.ff.net.2,up_blocks.1.attentions.2.transformer_blocks.1.attn1.to_k,up_blocks.1.attentions.2.transformer_blocks.1.attn1.to_out.0,up_blocks.1.attentions.2.transformer_blocks.1.attn1.to_q,up_blocks.1.attentions.2.transformer_blocks.1.attn1.to_v,up_blocks.1.attentions.2.transformer_blocks.1.attn2.to_k,up_blocks.1.attentions.2.transformer_blocks.1.attn2.to_out.0,up_blocks.1.attentions.2.transformer_blocks.1.attn2.to_q,up_blocks.1.attentions.2.transformer_blocks.1.attn2.to_v,up_blocks.1.attentions.2.transformer_blocks.1.ff.net.0.proj,up_blocks.1.attentions.2.transformer_blocks.1.ff.net.2" \
+  --lora_rank 32 \
+  --use_gradient_checkpointing \
+  --align_to_opensource_format

examples/stable_diffusion_xl/model_training/special/split_training/stable-diffusion-xl-base-1.0.sh

@@ -0,0 +1,40 @@
+modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "stable_diffusion_xl/stable-diffusion-xl-base-1.0/*" --local_dir ./data/diffsynth_example_dataset
+
+# Stage 1: cache deterministic preprocessing outputs.
+accelerate launch examples/stable_diffusion_xl/model_training/train.py \
+  --dataset_base_path data/diffsynth_example_dataset/stable_diffusion_xl/stable-diffusion-xl-base-1.0 \
+  --dataset_metadata_path data/diffsynth_example_dataset/stable_diffusion_xl/stable-diffusion-xl-base-1.0/metadata.csv \
+  --height 1024 \
+  --width 1024 \
+  --dataset_repeat 1 \
+  --model_id_with_origin_paths stabilityai/stable-diffusion-xl-base-1.0:text_encoder/model.safetensors,stabilityai/stable-diffusion-xl-base-1.0:text_encoder_2/model.safetensors,stabilityai/stable-diffusion-xl-base-1.0:unet/diffusion_pytorch_model.safetensors,stabilityai/stable-diffusion-xl-base-1.0:vae/diffusion_pytorch_model.safetensors \
+  --learning_rate 1e-4 \
+  --num_epochs 5 \
+  --remove_prefix_in_ckpt pipe.unet. \
+  --output_path ./models/train/stable-diffusion-xl-base-1.0_split_cache \
+  --lora_base_model unet \
+  --lora_target_modules mid_block.attentions.0.proj_in,mid_block.attentions.0.proj_out,down_blocks.1.attentions.0.proj_in,down_blocks.1.attentions.0.proj_out,down_blocks.1.attentions.0.transformer_blocks.0.attn1.to_k,down_blocks.1.attentions.0.transformer_blocks.0.attn1.to_out.0,down_blocks.1.attentions.0.transformer_blocks.0.attn1.to_q,down_blocks.1.attentions.0.transformer_blocks.0.attn1.to_v,down_blocks.1.attentions.0.transformer_blocks.0.attn2.to_k,down_blocks.1.attentions.0.transformer_blocks.0.attn2.to_out.0,down_blocks.1.attentions.0.transformer_blocks.0.attn2.to_q,down_blocks.1.attentions.0.transformer_blocks.0.attn2.to_v,down_blocks.1.attentions.0.transformer_blocks.0.ff.net.0.proj,down_blocks.1.attentions.0.transformer_blocks.0.ff.net.2,down_blocks.1.attentions.0.transformer_blocks.1.attn1.to_k,down_blocks.1.attentions.0.transformer_blocks.1.attn1.to_out.0,down_blocks.1.attentions.0.transformer_blocks.1.attn1.to_q,down_blocks.1.attentions.0.transformer_blocks.1.attn1.to_v,down_blocks.1.attentions.0.transformer_blocks.1.attn2.to_k,down_blocks.1.attentions.0.transformer_blocks.1.attn2.to_out.0,down_blocks.1.attentions.0.transformer_blocks.1.attn2.to_q,down_blocks.1.attentions.0.transformer_blocks.1.attn2.to_v,down_blocks.1.attentions.0.transformer_blocks.1.ff.net.0.proj,down_blocks.1.attentions.0.transformer_blocks.1.ff.net.2,down_blocks.1.attentions.1.proj_in,down_blocks.1.attentions.1.proj_out,down_blocks.1.attentions.1.transformer_blocks.0.attn1.to_k,down_blocks.1.attentions.1.transformer_blocks.0.attn1.to_out.0,down_blocks.1.attentions.1.transformer_blocks.0.attn1.to_q,down_blocks.1.attentions.1.transformer_blocks.0.attn1.to_v,down_blocks.1.attentions.1.transformer_blocks.0.attn2.to_k,down_blocks.1.attentions.1.transformer_blocks.0.attn2.to_out.0,down_blocks.1.attentions.1.transformer_blocks.0.attn2.to_q,down_blocks.1.attentions.1.transformer_blocks.0.attn2.to_v,down_blocks.1.attentions.1.transformer_blocks.0.ff.net.0.proj,down_blocks.1.attentions.1.transformer_blocks.0.ff.net.2,down_blocks.1.attentions.1.transformer_blocks.1.attn1.to_k,down_blocks.1.attentions.1.transformer_blocks.1.attn1.to_out.0,down_blocks.1.attentions.1.transformer_blocks.1.attn1.to_q,down_blocks.1.attentions.1.transformer_blocks.1.attn1.to_v,down_blocks.1.attentions.1.transformer_blocks.1.attn2.to_k,down_blocks.1.attentions.1.transformer_blocks.1.attn2.to_out.0,down_blocks.1.attentions.1.transformer_blocks.1.attn2.to_q,down_blocks.1.attentions.1.transformer_blocks.1.attn2.to_v,down_blocks.1.attentions.1.transformer_blocks.1.ff.net.0.proj,down_blocks.1.attentions.1.transformer_blocks.1.ff.net.2,down_blocks.2.attentions.0.proj_in,down_blocks.2.attentions.0.proj_out,down_blocks.2.attentions.0.transformer_blocks.0.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.0.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.0.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.0.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.0.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.0.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.0.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.0.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.0.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.0.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.1.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.1.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.1.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.1.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.1.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.1.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.1.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.1.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.1.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.1.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.2.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.2.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.2.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.2.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.2.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.2.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.2.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.2.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.2.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.2.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.3.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.3.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.3.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.3.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.3.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.3.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.3.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.3.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.3.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.3.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.4.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.4.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.4.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.4.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.4.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.4.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.4.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.4.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.4.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.4.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.5.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.5.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.5.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.5.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.5.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.5.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.5.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.5.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.5.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.5.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.6.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.6.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.6.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.6.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.6.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.6.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.6.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.6.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.6.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.6.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.7.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.7.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.7.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.7.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.7.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.7.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.7.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.7.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.7.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.7.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.8.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.8.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.8.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.8.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.8.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.8.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.8.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.8.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.8.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.8.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.9.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.9.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.9.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.9.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.9.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.9.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.9.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.9.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.9.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.9.ff.net.2,down_blocks.2.attentions.1.proj_in,down_blocks.2.attentions.1.proj_out,down_blocks.2.attentions.1.transformer_blocks.0.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.0.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.0.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.0.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.0.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.0.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.0.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.0.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.0.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.0.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.1.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.1.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.1.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.1.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.1.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.1.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.1.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.1.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.1.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.1.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.2.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.2.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.2.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.2.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.2.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.2.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.2.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.2.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.2.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.2.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.3.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.3.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.3.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.3.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.3.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.3.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.3.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.3.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.3.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.3.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.4.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.4.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.4.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.4.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.4.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.4.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.4.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.4.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.4.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.4.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.5.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.5.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.5.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.5.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.5.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.5.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.5.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.5.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.5.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.5.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.6.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.6.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.6.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.6.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.6.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.6.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.6.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.6.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.6.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.6.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.7.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.7.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.7.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.7.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.7.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.7.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.7.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.7.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.7.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.7.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.8.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.8.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.8.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.8.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.8.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.8.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.8.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.8.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.8.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.8.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.9.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.9.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.9.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.9.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.9.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.9.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.9.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.9.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.9.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.9.ff.net.2,mid_block.attentions.0.transformer_blocks.0.attn1.to_k,mid_block.attentions.0.transformer_blocks.0.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.0.attn1.to_q,mid_block.attentions.0.transformer_blocks.0.attn1.to_v,mid_block.attentions.0.transformer_blocks.0.attn2.to_k,mid_block.attentions.0.transformer_blocks.0.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.0.attn2.to_q,mid_block.attentions.0.transformer_blocks.0.attn2.to_v,mid_block.attentions.0.transformer_blocks.0.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.0.ff.net.2,mid_block.attentions.0.transformer_blocks.1.attn1.to_k,mid_block.attentions.0.transformer_blocks.1.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.1.attn1.to_q,mid_block.attentions.0.transformer_blocks.1.attn1.to_v,mid_block.attentions.0.transformer_blocks.1.attn2.to_k,mid_block.attentions.0.transformer_blocks.1.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.1.attn2.to_q,mid_block.attentions.0.transformer_blocks.1.attn2.to_v,mid_block.attentions.0.transformer_blocks.1.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.1.ff.net.2,mid_block.attentions.0.transformer_blocks.2.attn1.to_k,mid_block.attentions.0.transformer_blocks.2.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.2.attn1.to_q,mid_block.attentions.0.transformer_blocks.2.attn1.to_v,mid_block.attentions.0.transformer_blocks.2.attn2.to_k,mid_block.attentions.0.transformer_blocks.2.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.2.attn2.to_q,mid_block.attentions.0.transformer_blocks.2.attn2.to_v,mid_block.attentions.0.transformer_blocks.2.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.2.ff.net.2,mid_block.attentions.0.transformer_blocks.3.attn1.to_k,mid_block.attentions.0.transformer_blocks.3.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.3.attn1.to_q,mid_block.attentions.0.transformer_blocks.3.attn1.to_v,mid_block.attentions.0.transformer_blocks.3.attn2.to_k,mid_block.attentions.0.transformer_blocks.3.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.3.attn2.to_q,mid_block.attentions.0.transformer_blocks.3.attn2.to_v,mid_block.attentions.0.transformer_blocks.3.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.3.ff.net.2,mid_block.attentions.0.transformer_blocks.4.attn1.to_k,mid_block.attentions.0.transformer_blocks.4.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.4.attn1.to_q,mid_block.attentions.0.transformer_blocks.4.attn1.to_v,mid_block.attentions.0.transformer_blocks.4.attn2.to_k,mid_block.attentions.0.transformer_blocks.4.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.4.attn2.to_q,mid_block.attentions.0.transformer_blocks.4.attn2.to_v,mid_block.attentions.0.transformer_blocks.4.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.4.ff.net.2,mid_block.attentions.0.transformer_blocks.5.attn1.to_k,mid_block.attentions.0.transformer_blocks.5.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.5.attn1.to_q,mid_block.attentions.0.transformer_blocks.5.attn1.to_v,mid_block.attentions.0.transformer_blocks.5.attn2.to_k,mid_block.attentions.0.transformer_blocks.5.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.5.attn2.to_q,mid_block.attentions.0.transformer_blocks.5.attn2.to_v,mid_block.attentions.0.transformer_blocks.5.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.5.ff.net.2,mid_block.attentions.0.transformer_blocks.6.attn1.to_k,mid_block.attentions.0.transformer_blocks.6.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.6.attn1.to_q,mid_block.attentions.0.transformer_blocks.6.attn1.to_v,mid_block.attentions.0.transformer_blocks.6.attn2.to_k,mid_block.attentions.0.transformer_blocks.6.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.6.attn2.to_q,mid_block.attentions.0.transformer_blocks.6.attn2.to_v,mid_block.attentions.0.transformer_blocks.6.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.6.ff.net.2,mid_block.attentions.0.transformer_blocks.7.attn1.to_k,mid_block.attentions.0.transformer_blocks.7.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.7.attn1.to_q,mid_block.attentions.0.transformer_blocks.7.attn1.to_v,mid_block.attentions.0.transformer_blocks.7.attn2.to_k,mid_block.attentions.0.transformer_blocks.7.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.7.attn2.to_q,mid_block.attentions.0.transformer_blocks.7.attn2.to_v,mid_block.attentions.0.transformer_blocks.7.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.7.ff.net.2,mid_block.attentions.0.transformer_blocks.8.attn1.to_k,mid_block.attentions.0.transformer_blocks.8.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.8.attn1.to_q,mid_block.attentions.0.transformer_blocks.8.attn1.to_v,mid_block.attentions.0.transformer_blocks.8.attn2.to_k,mid_block.attentions.0.transformer_blocks.8.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.8.attn2.to_q,mid_block.attentions.0.transformer_blocks.8.attn2.to_v,mid_block.attentions.0.transformer_blocks.8.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.8.ff.net.2,mid_block.attentions.0.transformer_blocks.9.attn1.to_k,mid_block.attentions.0.transformer_blocks.9.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.9.attn1.to_q,mid_block.attentions.0.transformer_blocks.9.attn1.to_v,mid_block.attentions.0.transformer_blocks.9.attn2.to_k,mid_block.attentions.0.transformer_blocks.9.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.9.attn2.to_q,mid_block.attentions.0.transformer_blocks.9.attn2.to_v,mid_block.attentions.0.transformer_blocks.9.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.9.ff.net.2,up_blocks.0.attentions.0.proj_in,up_blocks.0.attentions.0.proj_out,up_blocks.0.attentions.0.transformer_blocks.0.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.0.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.0.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.0.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.0.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.0.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.0.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.0.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.0.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.0.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.1.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.1.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.1.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.1.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.1.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.1.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.1.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.1.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.1.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.1.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.2.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.2.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.2.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.2.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.2.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.2.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.2.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.2.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.2.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.2.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.3.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.3.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.3.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.3.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.3.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.3.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.3.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.3.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.3.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.3.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.4.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.4.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.4.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.4.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.4.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.4.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.4.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.4.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.4.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.4.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.5.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.5.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.5.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.5.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.5.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.5.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.5.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.5.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.5.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.5.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.6.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.6.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.6.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.6.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.6.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.6.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.6.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.6.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.6.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.6.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.7.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.7.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.7.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.7.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.7.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.7.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.7.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.7.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.7.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.7.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.8.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.8.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.8.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.8.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.8.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.8.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.8.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.8.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.8.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.8.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.9.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.9.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.9.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.9.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.9.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.9.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.9.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.9.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.9.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.9.ff.net.2,up_blocks.0.attentions.1.proj_in,up_blocks.0.attentions.1.proj_out,up_blocks.0.attentions.1.transformer_blocks.0.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.0.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.0.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.0.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.0.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.0.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.0.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.0.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.0.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.0.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.1.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.1.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.1.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.1.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.1.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.1.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.1.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.1.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.1.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.1.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.2.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.2.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.2.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.2.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.2.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.2.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.2.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.2.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.2.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.2.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.3.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.3.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.3.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.3.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.3.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.3.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.3.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.3.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.3.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.3.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.4.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.4.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.4.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.4.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.4.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.4.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.4.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.4.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.4.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.4.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.5.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.5.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.5.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.5.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.5.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.5.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.5.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.5.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.5.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.5.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.6.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.6.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.6.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.6.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.6.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.6.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.6.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.6.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.6.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.6.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.7.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.7.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.7.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.7.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.7.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.7.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.7.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.7.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.7.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.7.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.8.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.8.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.8.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.8.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.8.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.8.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.8.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.8.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.8.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.8.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.9.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.9.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.9.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.9.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.9.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.9.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.9.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.9.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.9.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.9.ff.net.2,up_blocks.0.attentions.2.proj_in,up_blocks.0.attentions.2.proj_out,up_blocks.0.attentions.2.transformer_blocks.0.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.0.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.0.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.0.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.0.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.0.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.0.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.0.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.0.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.0.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.1.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.1.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.1.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.1.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.1.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.1.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.1.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.1.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.1.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.1.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.2.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.2.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.2.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.2.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.2.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.2.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.2.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.2.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.2.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.2.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.3.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.3.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.3.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.3.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.3.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.3.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.3.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.3.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.3.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.3.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.4.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.4.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.4.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.4.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.4.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.4.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.4.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.4.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.4.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.4.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.5.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.5.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.5.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.5.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.5.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.5.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.5.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.5.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.5.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.5.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.6.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.6.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.6.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.6.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.6.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.6.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.6.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.6.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.6.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.6.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.7.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.7.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.7.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.7.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.7.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.7.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.7.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.7.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.7.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.7.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.8.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.8.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.8.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.8.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.8.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.8.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.8.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.8.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.8.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.8.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.9.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.9.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.9.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.9.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.9.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.9.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.9.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.9.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.9.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.9.ff.net.2,up_blocks.1.attentions.0.proj_in,up_blocks.1.attentions.0.proj_out,up_blocks.1.attentions.0.transformer_blocks.0.attn1.to_k,up_blocks.1.attentions.0.transformer_blocks.0.attn1.to_out.0,up_blocks.1.attentions.0.transformer_blocks.0.attn1.to_q,up_blocks.1.attentions.0.transformer_blocks.0.attn1.to_v,up_blocks.1.attentions.0.transformer_blocks.0.attn2.to_k,up_blocks.1.attentions.0.transformer_blocks.0.attn2.to_out.0,up_blocks.1.attentions.0.transformer_blocks.0.attn2.to_q,up_blocks.1.attentions.0.transformer_blocks.0.attn2.to_v,up_blocks.1.attentions.0.transformer_blocks.0.ff.net.0.proj,up_blocks.1.attentions.0.transformer_blocks.0.ff.net.2,up_blocks.1.attentions.0.transformer_blocks.1.attn1.to_k,up_blocks.1.attentions.0.transformer_blocks.1.attn1.to_out.0,up_blocks.1.attentions.0.transformer_blocks.1.attn1.to_q,up_blocks.1.attentions.0.transformer_blocks.1.attn1.to_v,up_blocks.1.attentions.0.transformer_blocks.1.attn2.to_k,up_blocks.1.attentions.0.transformer_blocks.1.attn2.to_out.0,up_blocks.1.attentions.0.transformer_blocks.1.attn2.to_q,up_blocks.1.attentions.0.transformer_blocks.1.attn2.to_v,up_blocks.1.attentions.0.transformer_blocks.1.ff.net.0.proj,up_blocks.1.attentions.0.transformer_blocks.1.ff.net.2,up_blocks.1.attentions.1.proj_in,up_blocks.1.attentions.1.proj_out,up_blocks.1.attentions.1.transformer_blocks.0.attn1.to_k,up_blocks.1.attentions.1.transformer_blocks.0.attn1.to_out.0,up_blocks.1.attentions.1.transformer_blocks.0.attn1.to_q,up_blocks.1.attentions.1.transformer_blocks.0.attn1.to_v,up_blocks.1.attentions.1.transformer_blocks.0.attn2.to_k,up_blocks.1.attentions.1.transformer_blocks.0.attn2.to_out.0,up_blocks.1.attentions.1.transformer_blocks.0.attn2.to_q,up_blocks.1.attentions.1.transformer_blocks.0.attn2.to_v,up_blocks.1.attentions.1.transformer_blocks.0.ff.net.0.proj,up_blocks.1.attentions.1.transformer_blocks.0.ff.net.2,up_blocks.1.attentions.1.transformer_blocks.1.attn1.to_k,up_blocks.1.attentions.1.transformer_blocks.1.attn1.to_out.0,up_blocks.1.attentions.1.transformer_blocks.1.attn1.to_q,up_blocks.1.attentions.1.transformer_blocks.1.attn1.to_v,up_blocks.1.attentions.1.transformer_blocks.1.attn2.to_k,up_blocks.1.attentions.1.transformer_blocks.1.attn2.to_out.0,up_blocks.1.attentions.1.transformer_blocks.1.attn2.to_q,up_blocks.1.attentions.1.transformer_blocks.1.attn2.to_v,up_blocks.1.attentions.1.transformer_blocks.1.ff.net.0.proj,up_blocks.1.attentions.1.transformer_blocks.1.ff.net.2,up_blocks.1.attentions.2.proj_in,up_blocks.1.attentions.2.proj_out,up_blocks.1.attentions.2.transformer_blocks.0.attn1.to_k,up_blocks.1.attentions.2.transformer_blocks.0.attn1.to_out.0,up_blocks.1.attentions.2.transformer_blocks.0.attn1.to_q,up_blocks.1.attentions.2.transformer_blocks.0.attn1.to_v,up_blocks.1.attentions.2.transformer_blocks.0.attn2.to_k,up_blocks.1.attentions.2.transformer_blocks.0.attn2.to_out.0,up_blocks.1.attentions.2.transformer_blocks.0.attn2.to_q,up_blocks.1.attentions.2.transformer_blocks.0.attn2.to_v,up_blocks.1.attentions.2.transformer_blocks.0.ff.net.0.proj,up_blocks.1.attentions.2.transformer_blocks.0.ff.net.2,up_blocks.1.attentions.2.transformer_blocks.1.attn1.to_k,up_blocks.1.attentions.2.transformer_blocks.1.attn1.to_out.0,up_blocks.1.attentions.2.transformer_blocks.1.attn1.to_q,up_blocks.1.attentions.2.transformer_blocks.1.attn1.to_v,up_blocks.1.attentions.2.transformer_blocks.1.attn2.to_k,up_blocks.1.attentions.2.transformer_blocks.1.attn2.to_out.0,up_blocks.1.attentions.2.transformer_blocks.1.attn2.to_q,up_blocks.1.attentions.2.transformer_blocks.1.attn2.to_v,up_blocks.1.attentions.2.transformer_blocks.1.ff.net.0.proj,up_blocks.1.attentions.2.transformer_blocks.1.ff.net.2 \
+  --lora_rank 32 \
+  --use_gradient_checkpointing \
+  --align_to_opensource_format \
+  --offload_models stabilityai/stable-diffusion-xl-base-1.0:unet/diffusion_pytorch_model.safetensors \
+  --task sft:data_process
+
+# Stage 2: train LoRA from the cached dataset.
+accelerate launch examples/stable_diffusion_xl/model_training/train.py \
+  --dataset_base_path ./models/train/stable-diffusion-xl-base-1.0_split_cache \
+  --height 1024 \
+  --width 1024 \
+  --dataset_repeat 10 \
+  --model_id_with_origin_paths stabilityai/stable-diffusion-xl-base-1.0:text_encoder/model.safetensors,stabilityai/stable-diffusion-xl-base-1.0:text_encoder_2/model.safetensors,stabilityai/stable-diffusion-xl-base-1.0:unet/diffusion_pytorch_model.safetensors,stabilityai/stable-diffusion-xl-base-1.0:vae/diffusion_pytorch_model.safetensors \
+  --learning_rate 1e-4 \
+  --num_epochs 5 \
+  --remove_prefix_in_ckpt pipe.unet. \
+  --output_path ./models/train/stable-diffusion-xl-base-1.0_split \
+  --lora_base_model unet \
+  --lora_target_modules mid_block.attentions.0.proj_in,mid_block.attentions.0.proj_out,down_blocks.1.attentions.0.proj_in,down_blocks.1.attentions.0.proj_out,down_blocks.1.attentions.0.transformer_blocks.0.attn1.to_k,down_blocks.1.attentions.0.transformer_blocks.0.attn1.to_out.0,down_blocks.1.attentions.0.transformer_blocks.0.attn1.to_q,down_blocks.1.attentions.0.transformer_blocks.0.attn1.to_v,down_blocks.1.attentions.0.transformer_blocks.0.attn2.to_k,down_blocks.1.attentions.0.transformer_blocks.0.attn2.to_out.0,down_blocks.1.attentions.0.transformer_blocks.0.attn2.to_q,down_blocks.1.attentions.0.transformer_blocks.0.attn2.to_v,down_blocks.1.attentions.0.transformer_blocks.0.ff.net.0.proj,down_blocks.1.attentions.0.transformer_blocks.0.ff.net.2,down_blocks.1.attentions.0.transformer_blocks.1.attn1.to_k,down_blocks.1.attentions.0.transformer_blocks.1.attn1.to_out.0,down_blocks.1.attentions.0.transformer_blocks.1.attn1.to_q,down_blocks.1.attentions.0.transformer_blocks.1.attn1.to_v,down_blocks.1.attentions.0.transformer_blocks.1.attn2.to_k,down_blocks.1.attentions.0.transformer_blocks.1.attn2.to_out.0,down_blocks.1.attentions.0.transformer_blocks.1.attn2.to_q,down_blocks.1.attentions.0.transformer_blocks.1.attn2.to_v,down_blocks.1.attentions.0.transformer_blocks.1.ff.net.0.proj,down_blocks.1.attentions.0.transformer_blocks.1.ff.net.2,down_blocks.1.attentions.1.proj_in,down_blocks.1.attentions.1.proj_out,down_blocks.1.attentions.1.transformer_blocks.0.attn1.to_k,down_blocks.1.attentions.1.transformer_blocks.0.attn1.to_out.0,down_blocks.1.attentions.1.transformer_blocks.0.attn1.to_q,down_blocks.1.attentions.1.transformer_blocks.0.attn1.to_v,down_blocks.1.attentions.1.transformer_blocks.0.attn2.to_k,down_blocks.1.attentions.1.transformer_blocks.0.attn2.to_out.0,down_blocks.1.attentions.1.transformer_blocks.0.attn2.to_q,down_blocks.1.attentions.1.transformer_blocks.0.attn2.to_v,down_blocks.1.attentions.1.transformer_blocks.0.ff.net.0.proj,down_blocks.1.attentions.1.transformer_blocks.0.ff.net.2,down_blocks.1.attentions.1.transformer_blocks.1.attn1.to_k,down_blocks.1.attentions.1.transformer_blocks.1.attn1.to_out.0,down_blocks.1.attentions.1.transformer_blocks.1.attn1.to_q,down_blocks.1.attentions.1.transformer_blocks.1.attn1.to_v,down_blocks.1.attentions.1.transformer_blocks.1.attn2.to_k,down_blocks.1.attentions.1.transformer_blocks.1.attn2.to_out.0,down_blocks.1.attentions.1.transformer_blocks.1.attn2.to_q,down_blocks.1.attentions.1.transformer_blocks.1.attn2.to_v,down_blocks.1.attentions.1.transformer_blocks.1.ff.net.0.proj,down_blocks.1.attentions.1.transformer_blocks.1.ff.net.2,down_blocks.2.attentions.0.proj_in,down_blocks.2.attentions.0.proj_out,down_blocks.2.attentions.0.transformer_blocks.0.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.0.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.0.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.0.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.0.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.0.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.0.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.0.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.0.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.0.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.1.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.1.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.1.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.1.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.1.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.1.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.1.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.1.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.1.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.1.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.2.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.2.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.2.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.2.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.2.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.2.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.2.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.2.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.2.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.2.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.3.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.3.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.3.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.3.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.3.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.3.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.3.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.3.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.3.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.3.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.4.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.4.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.4.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.4.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.4.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.4.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.4.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.4.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.4.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.4.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.5.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.5.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.5.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.5.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.5.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.5.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.5.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.5.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.5.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.5.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.6.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.6.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.6.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.6.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.6.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.6.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.6.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.6.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.6.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.6.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.7.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.7.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.7.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.7.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.7.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.7.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.7.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.7.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.7.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.7.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.8.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.8.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.8.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.8.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.8.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.8.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.8.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.8.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.8.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.8.ff.net.2,down_blocks.2.attentions.0.transformer_blocks.9.attn1.to_k,down_blocks.2.attentions.0.transformer_blocks.9.attn1.to_out.0,down_blocks.2.attentions.0.transformer_blocks.9.attn1.to_q,down_blocks.2.attentions.0.transformer_blocks.9.attn1.to_v,down_blocks.2.attentions.0.transformer_blocks.9.attn2.to_k,down_blocks.2.attentions.0.transformer_blocks.9.attn2.to_out.0,down_blocks.2.attentions.0.transformer_blocks.9.attn2.to_q,down_blocks.2.attentions.0.transformer_blocks.9.attn2.to_v,down_blocks.2.attentions.0.transformer_blocks.9.ff.net.0.proj,down_blocks.2.attentions.0.transformer_blocks.9.ff.net.2,down_blocks.2.attentions.1.proj_in,down_blocks.2.attentions.1.proj_out,down_blocks.2.attentions.1.transformer_blocks.0.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.0.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.0.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.0.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.0.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.0.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.0.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.0.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.0.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.0.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.1.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.1.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.1.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.1.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.1.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.1.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.1.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.1.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.1.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.1.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.2.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.2.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.2.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.2.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.2.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.2.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.2.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.2.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.2.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.2.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.3.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.3.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.3.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.3.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.3.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.3.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.3.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.3.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.3.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.3.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.4.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.4.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.4.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.4.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.4.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.4.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.4.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.4.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.4.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.4.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.5.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.5.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.5.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.5.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.5.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.5.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.5.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.5.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.5.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.5.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.6.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.6.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.6.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.6.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.6.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.6.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.6.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.6.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.6.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.6.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.7.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.7.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.7.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.7.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.7.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.7.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.7.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.7.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.7.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.7.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.8.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.8.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.8.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.8.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.8.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.8.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.8.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.8.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.8.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.8.ff.net.2,down_blocks.2.attentions.1.transformer_blocks.9.attn1.to_k,down_blocks.2.attentions.1.transformer_blocks.9.attn1.to_out.0,down_blocks.2.attentions.1.transformer_blocks.9.attn1.to_q,down_blocks.2.attentions.1.transformer_blocks.9.attn1.to_v,down_blocks.2.attentions.1.transformer_blocks.9.attn2.to_k,down_blocks.2.attentions.1.transformer_blocks.9.attn2.to_out.0,down_blocks.2.attentions.1.transformer_blocks.9.attn2.to_q,down_blocks.2.attentions.1.transformer_blocks.9.attn2.to_v,down_blocks.2.attentions.1.transformer_blocks.9.ff.net.0.proj,down_blocks.2.attentions.1.transformer_blocks.9.ff.net.2,mid_block.attentions.0.transformer_blocks.0.attn1.to_k,mid_block.attentions.0.transformer_blocks.0.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.0.attn1.to_q,mid_block.attentions.0.transformer_blocks.0.attn1.to_v,mid_block.attentions.0.transformer_blocks.0.attn2.to_k,mid_block.attentions.0.transformer_blocks.0.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.0.attn2.to_q,mid_block.attentions.0.transformer_blocks.0.attn2.to_v,mid_block.attentions.0.transformer_blocks.0.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.0.ff.net.2,mid_block.attentions.0.transformer_blocks.1.attn1.to_k,mid_block.attentions.0.transformer_blocks.1.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.1.attn1.to_q,mid_block.attentions.0.transformer_blocks.1.attn1.to_v,mid_block.attentions.0.transformer_blocks.1.attn2.to_k,mid_block.attentions.0.transformer_blocks.1.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.1.attn2.to_q,mid_block.attentions.0.transformer_blocks.1.attn2.to_v,mid_block.attentions.0.transformer_blocks.1.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.1.ff.net.2,mid_block.attentions.0.transformer_blocks.2.attn1.to_k,mid_block.attentions.0.transformer_blocks.2.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.2.attn1.to_q,mid_block.attentions.0.transformer_blocks.2.attn1.to_v,mid_block.attentions.0.transformer_blocks.2.attn2.to_k,mid_block.attentions.0.transformer_blocks.2.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.2.attn2.to_q,mid_block.attentions.0.transformer_blocks.2.attn2.to_v,mid_block.attentions.0.transformer_blocks.2.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.2.ff.net.2,mid_block.attentions.0.transformer_blocks.3.attn1.to_k,mid_block.attentions.0.transformer_blocks.3.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.3.attn1.to_q,mid_block.attentions.0.transformer_blocks.3.attn1.to_v,mid_block.attentions.0.transformer_blocks.3.attn2.to_k,mid_block.attentions.0.transformer_blocks.3.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.3.attn2.to_q,mid_block.attentions.0.transformer_blocks.3.attn2.to_v,mid_block.attentions.0.transformer_blocks.3.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.3.ff.net.2,mid_block.attentions.0.transformer_blocks.4.attn1.to_k,mid_block.attentions.0.transformer_blocks.4.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.4.attn1.to_q,mid_block.attentions.0.transformer_blocks.4.attn1.to_v,mid_block.attentions.0.transformer_blocks.4.attn2.to_k,mid_block.attentions.0.transformer_blocks.4.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.4.attn2.to_q,mid_block.attentions.0.transformer_blocks.4.attn2.to_v,mid_block.attentions.0.transformer_blocks.4.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.4.ff.net.2,mid_block.attentions.0.transformer_blocks.5.attn1.to_k,mid_block.attentions.0.transformer_blocks.5.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.5.attn1.to_q,mid_block.attentions.0.transformer_blocks.5.attn1.to_v,mid_block.attentions.0.transformer_blocks.5.attn2.to_k,mid_block.attentions.0.transformer_blocks.5.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.5.attn2.to_q,mid_block.attentions.0.transformer_blocks.5.attn2.to_v,mid_block.attentions.0.transformer_blocks.5.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.5.ff.net.2,mid_block.attentions.0.transformer_blocks.6.attn1.to_k,mid_block.attentions.0.transformer_blocks.6.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.6.attn1.to_q,mid_block.attentions.0.transformer_blocks.6.attn1.to_v,mid_block.attentions.0.transformer_blocks.6.attn2.to_k,mid_block.attentions.0.transformer_blocks.6.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.6.attn2.to_q,mid_block.attentions.0.transformer_blocks.6.attn2.to_v,mid_block.attentions.0.transformer_blocks.6.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.6.ff.net.2,mid_block.attentions.0.transformer_blocks.7.attn1.to_k,mid_block.attentions.0.transformer_blocks.7.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.7.attn1.to_q,mid_block.attentions.0.transformer_blocks.7.attn1.to_v,mid_block.attentions.0.transformer_blocks.7.attn2.to_k,mid_block.attentions.0.transformer_blocks.7.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.7.attn2.to_q,mid_block.attentions.0.transformer_blocks.7.attn2.to_v,mid_block.attentions.0.transformer_blocks.7.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.7.ff.net.2,mid_block.attentions.0.transformer_blocks.8.attn1.to_k,mid_block.attentions.0.transformer_blocks.8.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.8.attn1.to_q,mid_block.attentions.0.transformer_blocks.8.attn1.to_v,mid_block.attentions.0.transformer_blocks.8.attn2.to_k,mid_block.attentions.0.transformer_blocks.8.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.8.attn2.to_q,mid_block.attentions.0.transformer_blocks.8.attn2.to_v,mid_block.attentions.0.transformer_blocks.8.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.8.ff.net.2,mid_block.attentions.0.transformer_blocks.9.attn1.to_k,mid_block.attentions.0.transformer_blocks.9.attn1.to_out.0,mid_block.attentions.0.transformer_blocks.9.attn1.to_q,mid_block.attentions.0.transformer_blocks.9.attn1.to_v,mid_block.attentions.0.transformer_blocks.9.attn2.to_k,mid_block.attentions.0.transformer_blocks.9.attn2.to_out.0,mid_block.attentions.0.transformer_blocks.9.attn2.to_q,mid_block.attentions.0.transformer_blocks.9.attn2.to_v,mid_block.attentions.0.transformer_blocks.9.ff.net.0.proj,mid_block.attentions.0.transformer_blocks.9.ff.net.2,up_blocks.0.attentions.0.proj_in,up_blocks.0.attentions.0.proj_out,up_blocks.0.attentions.0.transformer_blocks.0.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.0.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.0.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.0.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.0.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.0.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.0.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.0.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.0.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.0.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.1.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.1.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.1.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.1.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.1.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.1.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.1.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.1.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.1.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.1.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.2.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.2.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.2.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.2.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.2.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.2.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.2.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.2.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.2.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.2.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.3.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.3.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.3.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.3.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.3.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.3.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.3.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.3.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.3.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.3.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.4.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.4.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.4.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.4.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.4.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.4.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.4.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.4.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.4.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.4.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.5.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.5.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.5.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.5.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.5.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.5.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.5.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.5.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.5.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.5.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.6.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.6.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.6.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.6.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.6.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.6.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.6.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.6.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.6.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.6.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.7.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.7.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.7.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.7.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.7.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.7.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.7.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.7.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.7.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.7.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.8.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.8.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.8.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.8.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.8.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.8.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.8.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.8.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.8.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.8.ff.net.2,up_blocks.0.attentions.0.transformer_blocks.9.attn1.to_k,up_blocks.0.attentions.0.transformer_blocks.9.attn1.to_out.0,up_blocks.0.attentions.0.transformer_blocks.9.attn1.to_q,up_blocks.0.attentions.0.transformer_blocks.9.attn1.to_v,up_blocks.0.attentions.0.transformer_blocks.9.attn2.to_k,up_blocks.0.attentions.0.transformer_blocks.9.attn2.to_out.0,up_blocks.0.attentions.0.transformer_blocks.9.attn2.to_q,up_blocks.0.attentions.0.transformer_blocks.9.attn2.to_v,up_blocks.0.attentions.0.transformer_blocks.9.ff.net.0.proj,up_blocks.0.attentions.0.transformer_blocks.9.ff.net.2,up_blocks.0.attentions.1.proj_in,up_blocks.0.attentions.1.proj_out,up_blocks.0.attentions.1.transformer_blocks.0.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.0.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.0.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.0.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.0.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.0.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.0.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.0.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.0.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.0.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.1.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.1.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.1.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.1.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.1.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.1.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.1.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.1.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.1.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.1.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.2.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.2.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.2.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.2.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.2.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.2.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.2.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.2.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.2.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.2.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.3.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.3.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.3.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.3.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.3.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.3.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.3.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.3.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.3.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.3.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.4.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.4.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.4.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.4.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.4.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.4.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.4.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.4.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.4.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.4.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.5.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.5.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.5.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.5.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.5.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.5.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.5.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.5.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.5.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.5.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.6.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.6.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.6.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.6.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.6.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.6.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.6.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.6.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.6.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.6.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.7.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.7.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.7.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.7.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.7.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.7.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.7.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.7.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.7.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.7.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.8.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.8.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.8.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.8.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.8.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.8.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.8.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.8.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.8.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.8.ff.net.2,up_blocks.0.attentions.1.transformer_blocks.9.attn1.to_k,up_blocks.0.attentions.1.transformer_blocks.9.attn1.to_out.0,up_blocks.0.attentions.1.transformer_blocks.9.attn1.to_q,up_blocks.0.attentions.1.transformer_blocks.9.attn1.to_v,up_blocks.0.attentions.1.transformer_blocks.9.attn2.to_k,up_blocks.0.attentions.1.transformer_blocks.9.attn2.to_out.0,up_blocks.0.attentions.1.transformer_blocks.9.attn2.to_q,up_blocks.0.attentions.1.transformer_blocks.9.attn2.to_v,up_blocks.0.attentions.1.transformer_blocks.9.ff.net.0.proj,up_blocks.0.attentions.1.transformer_blocks.9.ff.net.2,up_blocks.0.attentions.2.proj_in,up_blocks.0.attentions.2.proj_out,up_blocks.0.attentions.2.transformer_blocks.0.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.0.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.0.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.0.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.0.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.0.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.0.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.0.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.0.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.0.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.1.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.1.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.1.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.1.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.1.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.1.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.1.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.1.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.1.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.1.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.2.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.2.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.2.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.2.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.2.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.2.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.2.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.2.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.2.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.2.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.3.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.3.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.3.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.3.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.3.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.3.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.3.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.3.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.3.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.3.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.4.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.4.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.4.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.4.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.4.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.4.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.4.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.4.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.4.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.4.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.5.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.5.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.5.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.5.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.5.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.5.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.5.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.5.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.5.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.5.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.6.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.6.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.6.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.6.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.6.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.6.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.6.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.6.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.6.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.6.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.7.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.7.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.7.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.7.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.7.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.7.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.7.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.7.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.7.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.7.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.8.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.8.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.8.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.8.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.8.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.8.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.8.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.8.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.8.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.8.ff.net.2,up_blocks.0.attentions.2.transformer_blocks.9.attn1.to_k,up_blocks.0.attentions.2.transformer_blocks.9.attn1.to_out.0,up_blocks.0.attentions.2.transformer_blocks.9.attn1.to_q,up_blocks.0.attentions.2.transformer_blocks.9.attn1.to_v,up_blocks.0.attentions.2.transformer_blocks.9.attn2.to_k,up_blocks.0.attentions.2.transformer_blocks.9.attn2.to_out.0,up_blocks.0.attentions.2.transformer_blocks.9.attn2.to_q,up_blocks.0.attentions.2.transformer_blocks.9.attn2.to_v,up_blocks.0.attentions.2.transformer_blocks.9.ff.net.0.proj,up_blocks.0.attentions.2.transformer_blocks.9.ff.net.2,up_blocks.1.attentions.0.proj_in,up_blocks.1.attentions.0.proj_out,up_blocks.1.attentions.0.transformer_blocks.0.attn1.to_k,up_blocks.1.attentions.0.transformer_blocks.0.attn1.to_out.0,up_blocks.1.attentions.0.transformer_blocks.0.attn1.to_q,up_blocks.1.attentions.0.transformer_blocks.0.attn1.to_v,up_blocks.1.attentions.0.transformer_blocks.0.attn2.to_k,up_blocks.1.attentions.0.transformer_blocks.0.attn2.to_out.0,up_blocks.1.attentions.0.transformer_blocks.0.attn2.to_q,up_blocks.1.attentions.0.transformer_blocks.0.attn2.to_v,up_blocks.1.attentions.0.transformer_blocks.0.ff.net.0.proj,up_blocks.1.attentions.0.transformer_blocks.0.ff.net.2,up_blocks.1.attentions.0.transformer_blocks.1.attn1.to_k,up_blocks.1.attentions.0.transformer_blocks.1.attn1.to_out.0,up_blocks.1.attentions.0.transformer_blocks.1.attn1.to_q,up_blocks.1.attentions.0.transformer_blocks.1.attn1.to_v,up_blocks.1.attentions.0.transformer_blocks.1.attn2.to_k,up_blocks.1.attentions.0.transformer_blocks.1.attn2.to_out.0,up_blocks.1.attentions.0.transformer_blocks.1.attn2.to_q,up_blocks.1.attentions.0.transformer_blocks.1.attn2.to_v,up_blocks.1.attentions.0.transformer_blocks.1.ff.net.0.proj,up_blocks.1.attentions.0.transformer_blocks.1.ff.net.2,up_blocks.1.attentions.1.proj_in,up_blocks.1.attentions.1.proj_out,up_blocks.1.attentions.1.transformer_blocks.0.attn1.to_k,up_blocks.1.attentions.1.transformer_blocks.0.attn1.to_out.0,up_blocks.1.attentions.1.transformer_blocks.0.attn1.to_q,up_blocks.1.attentions.1.transformer_blocks.0.attn1.to_v,up_blocks.1.attentions.1.transformer_blocks.0.attn2.to_k,up_blocks.1.attentions.1.transformer_blocks.0.attn2.to_out.0,up_blocks.1.attentions.1.transformer_blocks.0.attn2.to_q,up_blocks.1.attentions.1.transformer_blocks.0.attn2.to_v,up_blocks.1.attentions.1.transformer_blocks.0.ff.net.0.proj,up_blocks.1.attentions.1.transformer_blocks.0.ff.net.2,up_blocks.1.attentions.1.transformer_blocks.1.attn1.to_k,up_blocks.1.attentions.1.transformer_blocks.1.attn1.to_out.0,up_blocks.1.attentions.1.transformer_blocks.1.attn1.to_q,up_blocks.1.attentions.1.transformer_blocks.1.attn1.to_v,up_blocks.1.attentions.1.transformer_blocks.1.attn2.to_k,up_blocks.1.attentions.1.transformer_blocks.1.attn2.to_out.0,up_blocks.1.attentions.1.transformer_blocks.1.attn2.to_q,up_blocks.1.attentions.1.transformer_blocks.1.attn2.to_v,up_blocks.1.attentions.1.transformer_blocks.1.ff.net.0.proj,up_blocks.1.attentions.1.transformer_blocks.1.ff.net.2,up_blocks.1.attentions.2.proj_in,up_blocks.1.attentions.2.proj_out,up_blocks.1.attentions.2.transformer_blocks.0.attn1.to_k,up_blocks.1.attentions.2.transformer_blocks.0.attn1.to_out.0,up_blocks.1.attentions.2.transformer_blocks.0.attn1.to_q,up_blocks.1.attentions.2.transformer_blocks.0.attn1.to_v,up_blocks.1.attentions.2.transformer_blocks.0.attn2.to_k,up_blocks.1.attentions.2.transformer_blocks.0.attn2.to_out.0,up_blocks.1.attentions.2.transformer_blocks.0.attn2.to_q,up_blocks.1.attentions.2.transformer_blocks.0.attn2.to_v,up_blocks.1.attentions.2.transformer_blocks.0.ff.net.0.proj,up_blocks.1.attentions.2.transformer_blocks.0.ff.net.2,up_blocks.1.attentions.2.transformer_blocks.1.attn1.to_k,up_blocks.1.attentions.2.transformer_blocks.1.attn1.to_out.0,up_blocks.1.attentions.2.transformer_blocks.1.attn1.to_q,up_blocks.1.attentions.2.transformer_blocks.1.attn1.to_v,up_blocks.1.attentions.2.transformer_blocks.1.attn2.to_k,up_blocks.1.attentions.2.transformer_blocks.1.attn2.to_out.0,up_blocks.1.attentions.2.transformer_blocks.1.attn2.to_q,up_blocks.1.attentions.2.transformer_blocks.1.attn2.to_v,up_blocks.1.attentions.2.transformer_blocks.1.ff.net.0.proj,up_blocks.1.attentions.2.transformer_blocks.1.ff.net.2 \
+  --lora_rank 32 \
+  --use_gradient_checkpointing \
+  --align_to_opensource_format \
+  --offload_models stabilityai/stable-diffusion-xl-base-1.0:vae/diffusion_pytorch_model.safetensors \
+  --task sft:train

examples/stable_diffusion_xl/model_training/special/split_training/validate.py

@@ -0,0 +1,27 @@
+import torch
+from diffsynth.core import ModelConfig
+from diffsynth.pipelines.stable_diffusion_xl import StableDiffusionXLPipeline
+
+pipe = StableDiffusionXLPipeline.from_pretrained(
+    torch_dtype=torch.float32,
+    model_configs=[
+        ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="text_encoder/model.safetensors"),
+        ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="text_encoder_2/model.safetensors"),
+        ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="unet/diffusion_pytorch_model.safetensors"),
+        ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"),
+    ],
+    tokenizer_config=ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="tokenizer/"),
+    tokenizer_2_config=ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="tokenizer_2/"),
+)
+pipe.load_lora(pipe.unet, './models/train/stable-diffusion-xl-base-1.0_split/epoch-4.safetensors')
+
+image = pipe(
+    prompt="a dog",
+    negative_prompt="",
+    cfg_scale=7.0,
+    height=1024,
+    width=1024,
+    seed=42,
+    num_inference_steps=50,
+)
+image.save('split_training_stable-diffusion-xl.jpg')

examples/stable_diffusion_xl/model_training/train.py

@@ -0,0 +1,159 @@
+import torch, os, argparse, accelerate
+from diffsynth.core import UnifiedDataset
+from diffsynth.pipelines.stable_diffusion_xl import StableDiffusionXLPipeline, ModelConfig
+from diffsynth.diffusion import *
+from diffsynth.utils.lora.sdxl import SdxlLoRAConverter
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+
+class StableDiffusionXLTrainingModule(DiffusionTrainingModule):
+    def __init__(
+        self,
+        model_paths=None, model_id_with_origin_paths=None,
+        tokenizer_path=None,
+        trainable_models=None,
+        lora_base_model=None, lora_target_modules="", lora_rank=32, lora_checkpoint=None,
+        preset_lora_path=None, preset_lora_model=None,
+        use_gradient_checkpointing=True,
+        use_gradient_checkpointing_offload=False,
+        extra_inputs=None,
+        fp8_models=None,
+        offload_models=None,
+        quant_options=None,
+        resume_from_checkpoint=None, remove_prefix_in_ckpt=None,
+        device="cpu",
+        task="sft",
+    ):
+        super().__init__()
+        # Load models
+        model_configs = self.parse_model_configs(model_paths, model_id_with_origin_paths, fp8_models=fp8_models, offload_models=offload_models, quant_options=quant_options, device=device)
+        tokenizer_config = self.parse_path_or_model_id(tokenizer_path, ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="tokenizer/"))
+        tokenizer_2_config = self.parse_path_or_model_id(tokenizer_path, ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="tokenizer_2/"))
+        self.pipe = StableDiffusionXLPipeline.from_pretrained(torch_dtype=torch.float32, device=device, model_configs=model_configs, tokenizer_config=tokenizer_config, tokenizer_2_config=tokenizer_2_config)
+        self.pipe = self.split_pipeline_units(task, self.pipe, trainable_models, lora_base_model)
+        self.resume_from_checkpoint(resume_from_checkpoint, remove_prefix_in_ckpt)
+
+        # Training mode
+        self.switch_pipe_to_training_mode(
+            self.pipe, trainable_models,
+            lora_base_model, lora_target_modules, lora_rank, lora_checkpoint,
+            preset_lora_path, preset_lora_model,
+            task=task,
+        )
+        
+        # Other configs
+        self.use_gradient_checkpointing = use_gradient_checkpointing
+        self.use_gradient_checkpointing_offload = use_gradient_checkpointing_offload
+        self.extra_inputs = extra_inputs.split(",") if extra_inputs is not None else []
+        self.fp8_models = fp8_models
+        self.task = task
+        self.task_to_loss = {
+            "sft:data_process": lambda pipe, *args: args,
+            "direct_distill:data_process": lambda pipe, *args: args,
+            "sft": lambda pipe, inputs_shared, inputs_posi, inputs_nega: FlowMatchSFTLoss(pipe, **inputs_shared, **inputs_posi),
+            "sft:train": lambda pipe, inputs_shared, inputs_posi, inputs_nega: FlowMatchSFTLoss(pipe, **inputs_shared, **inputs_posi),
+            "direct_distill": lambda pipe, inputs_shared, inputs_posi, inputs_nega: DirectDistillLoss(pipe, **inputs_shared, **inputs_posi),
+            "direct_distill:train": lambda pipe, inputs_shared, inputs_posi, inputs_nega: DirectDistillLoss(pipe, **inputs_shared, **inputs_posi),
+        }
+        
+    def get_pipeline_inputs(self, data):
+        inputs_posi = {"prompt": data["prompt"]}
+        inputs_nega = {"negative_prompt": ""}
+        inputs_shared = {
+            # Assume you are using this pipeline for inference,
+            # please fill in the input parameters.
+            "input_image": data["image"],
+            "height": data["image"].size[1],
+            "width": data["image"].size[0],
+            # Please do not modify the following parameters
+            # unless you clearly know what this will cause.
+            "cfg_scale": 1,
+            "rand_device": self.pipe.device,
+            "use_gradient_checkpointing": self.use_gradient_checkpointing,
+            "use_gradient_checkpointing_offload": self.use_gradient_checkpointing_offload,
+        }
+        inputs_shared = self.parse_extra_inputs(data, self.extra_inputs, inputs_shared)
+        return inputs_shared, inputs_posi, inputs_nega
+    
+    def forward(self, data, inputs=None):
+        if inputs is None: inputs = self.get_pipeline_inputs(data)
+        inputs = self.transfer_data_to_device(inputs, self.pipe.device, self.pipe.torch_dtype)
+        for unit in self.pipe.units:
+            inputs = self.pipe.unit_runner(unit, self.pipe, *inputs)
+        loss = self.task_to_loss[self.task](self.pipe, *inputs)
+        return loss
+
+
+def parser():
+    parser = argparse.ArgumentParser(description="Simple example of a training script.")
+    parser = add_general_config(parser)
+    parser = add_image_size_config(parser)
+    parser.add_argument("--tokenizer_path", type=str, default=None, help="Path to tokenizer.")
+    parser.add_argument("--tokenizer_2_path", type=str, default=None, help="Path to tokenizer 2.")
+    parser.add_argument("--align_to_opensource_format", default=False, action="store_true", help="Whether to align the lora format to opensource format.")
+    return parser
+
+
+if __name__ == "__main__":
+    parser = parser()
+    args = parser.parse_args()
+    accelerator = accelerate.Accelerator(
+        gradient_accumulation_steps=args.gradient_accumulation_steps,
+        kwargs_handlers=[accelerate.DistributedDataParallelKwargs(find_unused_parameters=args.find_unused_parameters)],
+    )
+    dataset = UnifiedDataset(
+        base_path=args.dataset_base_path,
+        metadata_path=args.dataset_metadata_path,
+        repeat=args.dataset_repeat,
+        data_file_keys=args.data_file_keys.split(","),
+        main_data_operator=UnifiedDataset.default_image_operator(
+            base_path=args.dataset_base_path,
+            max_pixels=args.max_pixels,
+            height=args.height,
+            width=args.width,
+            height_division_factor=32,
+            width_division_factor=32,
+        )
+    )
+    model = StableDiffusionXLTrainingModule(
+        model_paths=args.model_paths,
+        model_id_with_origin_paths=args.model_id_with_origin_paths,
+        tokenizer_path=args.tokenizer_path,
+        trainable_models=args.trainable_models,
+        lora_base_model=args.lora_base_model,
+        lora_target_modules=args.lora_target_modules,
+        lora_rank=args.lora_rank,
+        lora_checkpoint=args.lora_checkpoint,
+        preset_lora_path=args.preset_lora_path,
+        preset_lora_model=args.preset_lora_model,
+        use_gradient_checkpointing=args.use_gradient_checkpointing,
+        use_gradient_checkpointing_offload=args.use_gradient_checkpointing_offload,
+        extra_inputs=args.extra_inputs,
+        fp8_models=args.fp8_models,
+        offload_models=args.offload_models,
+        quant_options=args.quant_options,
+        resume_from_checkpoint=args.resume_from_checkpoint,
+        remove_prefix_in_ckpt=args.remove_prefix_in_ckpt,
+        task=args.task,
+        device="cpu" if args.enable_model_cpu_offload else accelerator.device,
+    )
+    model_logger = ModelLogger(
+        args.output_path,
+        remove_prefix_in_ckpt=args.remove_prefix_in_ckpt,
+        state_dict_converter=SdxlLoRAConverter.align_to_opensource_format if args.align_to_opensource_format else lambda x:x,
+        enable_tensorboard_log=args.enable_tensorboard_log,
+        enable_swanlab_log=args.enable_swanlab_log,
+        swanlab_project=args.swanlab_project,
+        enable_wandb_log=args.enable_wandb_log,
+        wandb_project=args.wandb_project,
+        enable_csv_log=args.enable_csv_log,
+    )
+    launcher_map = {
+        "sft:data_process": launch_data_process_task,
+        "direct_distill:data_process": launch_data_process_task,
+        "sft": launch_training_task,
+        "sft:train": launch_training_task,
+        "direct_distill": launch_training_task,
+        "direct_distill:train": launch_training_task,
+    }
+    launcher_map[args.task](accelerator, dataset, model, model_logger, args=args)

examples/stable_diffusion_xl/model_training/validate_full/stable-diffusion-xl-base-1.0.py

@@ -0,0 +1,28 @@
+from diffsynth.pipelines.stable_diffusion_xl import StableDiffusionXLPipeline, ModelConfig
+from diffsynth.core import load_state_dict
+import torch
+
+pipe = StableDiffusionXLPipeline.from_pretrained(
+    torch_dtype=torch.float32,
+    model_configs=[
+        ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="text_encoder/model.safetensors"),
+        ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="text_encoder_2/model.safetensors"),
+        ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="unet/diffusion_pytorch_model.safetensors"),
+        ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"),
+    ],
+    tokenizer_config=ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="tokenizer/"),
+    tokenizer_2_config=ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="tokenizer_2/"),
+)
+state_dict = load_state_dict("./models/train/stable-diffusion-xl-base-1.0_full/epoch-1.safetensors", torch_dtype=torch.float32)
+pipe.unet.load_state_dict(state_dict)
+
+image = pipe(
+    prompt="a dog",
+    negative_prompt="",
+    cfg_scale=7.0,
+    height=1024,
+    width=1024,
+    seed=42,
+    num_inference_steps=50,
+)
+image.save("image_stable-diffusion-xl-base-1.0_full.jpg")

examples/stable_diffusion_xl/model_training/validate_lora/stable-diffusion-xl-base-1.0.py

@@ -0,0 +1,27 @@
+import torch
+from diffsynth.core import ModelConfig
+from diffsynth.pipelines.stable_diffusion_xl import StableDiffusionXLPipeline
+
+pipe = StableDiffusionXLPipeline.from_pretrained(
+    torch_dtype=torch.float32,
+    model_configs=[
+        ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="text_encoder/model.safetensors"),
+        ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="text_encoder_2/model.safetensors"),
+        ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="unet/diffusion_pytorch_model.safetensors"),
+        ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"),
+    ],
+    tokenizer_config=ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="tokenizer/"),
+    tokenizer_2_config=ModelConfig(model_id="stabilityai/stable-diffusion-xl-base-1.0", origin_file_pattern="tokenizer_2/"),
+)
+pipe.load_lora(pipe.unet, "models/train/stable-diffusion-xl-base-1.0_lora/epoch-4.safetensors")
+
+image = pipe(
+    prompt="a dog",
+    negative_prompt="",
+    cfg_scale=7.0,
+    height=1024,
+    width=1024,
+    seed=42,
+    num_inference_steps=50,
+)
+image.save("image_stable-diffusion-xl-base-1.0.jpg")

examples/wanvideo/README.md

@@ -0,0 +1,3 @@
+English Document: https://diffsynth-studio-doc.readthedocs.io/en/latest/Model_Details/Wan.html
+
+中文文档：https://diffsynth-studio-doc.readthedocs.io/zh-cn/latest/Model_Details/Wan.html

examples/wanvideo/acceleration/Wan2.2-Animate-2-14B-usp.py

@@ -0,0 +1,117 @@
+import torch
+import torch.distributed as dist
+from PIL import Image
+from diffsynth.utils.data import save_video, VideoData
+from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig
+from modelscope import dataset_snapshot_download
+
+vram_config = {
+    "offload_dtype": torch.bfloat16,
+    "offload_device": "cpu",
+    "onload_dtype": torch.bfloat16,
+    "onload_device": "cuda",
+    "preparing_dtype": torch.bfloat16,
+    "preparing_device": "cuda",
+    "computation_dtype": torch.bfloat16,
+    "computation_device": "cuda",
+}
+
+pipe = WanVideoPipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    use_usp=True,
+    model_configs=[
+        ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors", **vram_config),
+        ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config),
+        ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config),
+        ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config),
+    ],
+    tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"),
+)
+
+# Character animation: reference image (identity) + reference video (motion) -> animated video.
+dataset_snapshot_download(
+    "DiffSynth-Studio/diffsynth_example_dataset",
+    local_dir="data/diffsynth_example_dataset",
+    allow_file_pattern="wanvideo/Wan2.2-Animate-2-14B/*"
+)
+reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B/refimage.jpg").convert("RGB")
+reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B/refvideo.mp4").raw_data()
+# Example 1: single-clip generation
+num_frames = 81
+video = pipe(
+    prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。",
+    negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走",
+    animate2_prompt_ref="视频中的人在做动作，背景静止",
+    animate2_reference_image=reference_image,
+    animate2_reference_video=reference_video[:num_frames],
+    animate2_offload_kv=True,
+    num_frames=num_frames, height=1280, width=720,
+    num_inference_steps=40, cfg_scale=3.0,
+    seed=0, tiled=True,
+)
+if dist.get_rank() == 0:
+    save_video(video, "video_Wan2.2-Animate-2-14B.mp4", fps=24, quality=5)
+
+
+# Example 2: multi-clip long-video generation
+def generate_long_video(pipe, reference_image, cond_images, clip_len, first_num=1, **kwargs):
+    assert clip_len > first_num, "clip_len must be greater than first_num"
+
+    def zigzag_padding(array, target_len):
+        if len(array) == 1:
+            return [array[0]] * target_len
+        idx, flip, out = 0, False, []
+        while len(out) < target_len:
+            out.append(array[idx])
+            idx += -1 if flip else 1
+            if idx == 0 or idx == len(array) - 1:
+                flip = not flip
+        return out[:target_len]
+
+    real_len = len(cond_images)
+    if real_len == 0:
+        return []
+    step = clip_len - first_num
+    # Precompute clip count so clips of `clip_len` stepping by `step` tile the (padded) driving video.
+    num_clips = 1 if real_len <= clip_len else (real_len - clip_len + step - 1) // step + 1
+    target_len = clip_len + (num_clips - 1) * step
+    if real_len < target_len:
+        cond_images = zigzag_padding(cond_images, target_len)
+
+    all_frames = []
+    prev_tail = None
+    for i in range(num_clips):
+        start = i * step
+        seg_driving = cond_images[start:start + clip_len]
+        seg_out = pipe(
+            animate2_reference_image=reference_image,
+            animate2_reference_video=seg_driving,
+            animate2_refert_images=None if i == 0 else prev_tail,
+            num_frames=clip_len,
+            **kwargs,
+        )
+        prev_tail = seg_out[-first_num:]
+        if i != 0:
+            seg_out = seg_out[first_num:]
+        all_frames.extend(seg_out)
+    return all_frames[:real_len]
+
+
+clip_len = 81
+long_video = generate_long_video(
+    pipe,
+    reference_image=reference_image,
+    cond_images=reference_video,
+    clip_len=clip_len,
+    first_num=1,
+    prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。",
+    negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走",
+    animate2_prompt_ref="视频中的人在做动作，背景静止",
+    animate2_offload_kv=True,
+    height=1280, width=720,
+    num_inference_steps=40, cfg_scale=3.0,
+    seed=0, tiled=True,
+)
+if dist.get_rank() == 0:
+    save_video(long_video, "video_Wan2.2-Animate-2-14B-long.mp4", fps=24, quality=5)

examples/wanvideo/acceleration/unified_sequence_parallel.py

@@ -0,0 +1,26 @@
+import torch
+from PIL import Image
+from diffsynth.utils.data import save_video, VideoData
+from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig
+import torch.distributed as dist
+
+pipe = WanVideoPipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    use_usp=True,
+    model_configs=[
+        ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="diffusion_pytorch_model*.safetensors"),
+        ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth"),
+        ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth"),
+    ],
+    tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="google/umt5-xxl/"),
+)
+
+# Text-to-video
+video = pipe(
+    prompt="一名宇航员身穿太空服，面朝镜头骑着一匹机械马在火星表面驰骋。红色的荒凉地表延伸至远方，点缀着巨大的陨石坑和奇特的岩石结构。机械马的步伐稳健，扬起微弱的尘埃，展现出未来科技与原始探索的完美结合。宇航员手持操控装置，目光坚定，仿佛正在开辟人类的新疆域。背景是深邃的宇宙和蔚蓝的地球，画面既科幻又充满希望，让人不禁畅想未来的星际生活。",
+    negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走",
+    seed=0, tiled=True,
+)
+if dist.get_rank() == 0:
+    save_video(video, "video1.mp4", fps=15, quality=5)

examples/wanvideo/model_inference/LongCat-Video.py

@@ -0,0 +1,35 @@
+import torch
+from diffsynth.utils.data import save_video, VideoData
+from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig
+
+
+pipe = WanVideoPipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="meituan-longcat/LongCat-Video", origin_file_pattern="dit/diffusion_pytorch_model*.safetensors"),
+        ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth"),
+        ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth"),
+    ],
+    tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="google/umt5-xxl/"),
+)
+
+# Text-to-video
+video = pipe(
+    prompt="In a realistic photography style, a white boy around seven or eight years old sits on a park bench, wearing a light blue T-shirt, denim shorts, and white sneakers. He holds an ice cream cone with vanilla and chocolate flavors, and beside him is a medium-sized golden Labrador. Smiling, the boy offers the ice cream to the dog, who eagerly licks it with its tongue. The sun is shining brightly, and the background features a green lawn and several tall trees, creating a warm and loving scene.",
+    negative_prompt="Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards",
+    seed=0, tiled=True, num_frames=93,
+    cfg_scale=2, sigma_shift=1,
+)
+save_video(video, "video_1_LongCat-Video.mp4", fps=15, quality=5)
+
+# Video-continuation (The number of frames in `longcat_video` should be 4n+1.)
+longcat_video = video[-17:]
+video = pipe(
+    prompt="In a realistic photography style, a white boy around seven or eight years old sits on a park bench, wearing a light blue T-shirt, denim shorts, and white sneakers. He holds an ice cream cone with vanilla and chocolate flavors, and beside him is a medium-sized golden Labrador. Smiling, the boy offers the ice cream to the dog, who eagerly licks it with its tongue. The sun is shining brightly, and the background features a green lawn and several tall trees, creating a warm and loving scene.",
+    negative_prompt="Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards",
+    seed=1, tiled=True, num_frames=93,
+    cfg_scale=2, sigma_shift=1,
+    longcat_video=longcat_video,
+)
+save_video(video, "video_2_LongCat-Video.mp4", fps=15, quality=5)

examples/wanvideo/model_inference/Video-As-Prompt-Wan2.1-14B.py

@@ -0,0 +1,49 @@
+import torch
+import PIL
+from PIL import Image
+from diffsynth.utils.data import save_video, VideoData
+from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig
+from modelscope import dataset_snapshot_download
+from typing import List
+
+
+pipe = WanVideoPipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="ByteDance/Video-As-Prompt-Wan2.1-14B", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"),
+        ModelConfig(model_id="Wan-AI/Wan2.1-I2V-14B-720P", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth"),
+        ModelConfig(model_id="Wan-AI/Wan2.1-I2V-14B-720P", origin_file_pattern="Wan2.1_VAE.pth"),
+        ModelConfig(model_id="Wan-AI/Wan2.1-I2V-14B-720P", origin_file_pattern="models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"),
+    ],
+    tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="google/umt5-xxl/"),
+)
+
+dataset_snapshot_download("DiffSynth-Studio/example_video_dataset", allow_file_pattern="wanvap/*", local_dir="data/example_video_dataset")
+ref_video_path = 'data/example_video_dataset/wanvap/vap_ref.mp4'
+target_image_path = 'data/example_video_dataset/wanvap/input_image.jpg'
+
+def select_frames(video_frames, num):
+    idx = torch.linspace(0, len(video_frames) - 1, num).long().tolist()
+    return [video_frames[i] for i in idx]
+
+image = Image.open(target_image_path).convert("RGB")
+ref_video = VideoData(ref_video_path, height=480, width=832)
+ref_frames = select_frames(ref_video, num=49)
+
+vap_prompt = "A man stands with his back to the camera on a dirt path overlooking sun-drenched, rolling green tea plantations. He wears a blue and green plaid shirt, dark pants, and white shoes. As he turns to face the camera and spreads his arms, a brief, magical burst of sparkling golden light particles envelops him. Through this shimmer, he seamlessly transforms into a Labubu toy character. His head morphs into the iconic large, furry-eared head of the toy, featuring a wide grin with pointed teeth and red cheek markings. The character retains the man's original plaid shirt and clothing, which now fit its stylized, cartoonish body. The camera remains static throughout the transformation, positioned low among the tea bushes, maintaining a consistent view of the subject and the expansive scenery."
+prompt = "A young woman with curly hair, wearing a green hijab and a floral dress, plays a violin in front of a vintage green car on a tree-lined street. She executes a swift counter-clockwise turn to face the camera. During the turn, a brilliant shower of golden, sparkling particles erupts and momentarily obscures her figure. As the particles fade, she is revealed to have seamlessly transformed into a Labubu toy character. This new figure, now with the toy's signature large ears, big eyes, and toothy grin, maintains the original pose and continues playing the violin. The character's clothing—the green hijab, floral dress, and black overcoat—remains identical to the woman's. Throughout this transition, the camera stays static, and the street-side environment remains completely consistent."
+negative_prompt = "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards"
+
+video = pipe(
+    prompt=prompt,
+    negative_prompt=negative_prompt,
+    input_image=image,
+    seed=42, tiled=True,
+    height=480, width=832,
+    num_frames=49,
+    vap_video=ref_frames,
+    vap_prompt=vap_prompt,
+    negative_vap_prompt=negative_prompt,
+)
+save_video(video, "video_Video-As-Prompt-Wan2.1-14B.mp4", fps=15, quality=5)

examples/wanvideo/model_inference/Wan-Dancer-14B-global.py

@@ -0,0 +1,48 @@
+import torch
+from PIL import Image
+from diffsynth.utils.data import save_video, VideoData
+from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig
+from modelscope import dataset_snapshot_download
+
+
+pipe = WanVideoPipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="Wan-AI/Wan-Dancer-14B", origin_file_pattern="global_model.safetensors"),
+        ModelConfig(model_id="Wan-AI/Wan-Dancer-14B", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth"),
+        ModelConfig(model_id="Wan-AI/Wan-Dancer-14B", origin_file_pattern="Wan2.1_VAE.pth"),
+        ModelConfig(model_id="Wan-AI/Wan-Dancer-14B", origin_file_pattern="models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"),
+    ],
+    tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="google/umt5-xxl/"),
+)
+dataset_snapshot_download(
+    "DiffSynth-Studio/diffsynth_example_dataset",
+    local_dir="data/diffsynth_example_dataset",
+    allow_file_pattern="wanvideo/Wan-Dancer-14B-global/*"
+)
+# This is a specialized model with the following constraints on its input parameters:
+# *   The model outputs a sequence of keyframes rather than a video; therefore, `framewise_decoding=True` must be set.
+# *   When the number of keyframes is $n$, `num_frames` = 4 * (n - 1) + 1.
+# *   Reducing `height`, `width`, `num_frames`, or `num_inference_steps` may lead to severe artifacts or generation failure.
+# *   The audio file specified by `wantodance_music_path` must match the video duration, calculated as (`num_frames` / 7.5) seconds.
+# *   The width and height of `wantodance_reference_image` must be multiples of 16.
+# *   `wantodance_fps` is configurable, but since the model appears to have been trained exclusively at 7.5 FPS, setting it to other values is not recommended.
+# *   The first frame of `wantodance_keyframes` is the `wantodance_reference_image`, while all subsequent frames are solid black.
+# *   `wantodance_keyframes_mask` indicates the positions of valid frames within `wantodance_keyframes`.
+wantodance_keyframes = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Dancer-14B-global/keyframes.mp4")
+wantodance_keyframes = [wantodance_keyframes[i] for i in range(149)]
+video = pipe(
+    prompt="一个人正在跳舞，舞蹈种类是韩舞。帧率是7.5000",
+    negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走",
+    seed=0, tiled=False,
+    height=1280, width=720, num_frames=149,
+    num_inference_steps=48,
+    wantodance_music_path="data/diffsynth_example_dataset/wanvideo/Wan-Dancer-14B-global/music.WAV",
+    wantodance_reference_image=Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Dancer-14B-global/refimage.jpg"),
+    wantodance_fps=7.5,
+    wantodance_keyframes=wantodance_keyframes,
+    wantodance_keyframes_mask=[1] + [0] * 148,
+    framewise_decoding=True,
+)
+save_video(video, "video_Wan-Dancer-14B-global.mp4", fps=7.5, quality=5)

examples/wanvideo/model_inference/Wan-Dancer-14B-local.py

@@ -0,0 +1,52 @@
+import torch, os
+from PIL import Image
+from diffsynth.utils.data import save_video, VideoData
+from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig
+from modelscope import dataset_snapshot_download
+
+
+pipe = WanVideoPipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="Wan-AI/Wan-Dancer-14B", origin_file_pattern="local_model.safetensors"),
+        ModelConfig(model_id="Wan-AI/Wan-Dancer-14B", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth"),
+        ModelConfig(model_id="Wan-AI/Wan-Dancer-14B", origin_file_pattern="Wan2.1_VAE.pth"),
+        ModelConfig(model_id="Wan-AI/Wan-Dancer-14B", origin_file_pattern="models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"),
+    ],
+    tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="google/umt5-xxl/"),
+)
+dataset_snapshot_download(
+    "DiffSynth-Studio/diffsynth_example_dataset",
+    local_dir="data/diffsynth_example_dataset",
+    allow_file_pattern="wanvideo/Wan-Dancer-14B-local/*"
+)
+# This is a specialized model with the following constraints on its input parameters:
+# *   The model renders and outputs video based on a sequence of keyframes; therefore, `wantodance_keyframes` must be provided correctly.
+# *   If you need to generate a long video, please generate it in segments, and ensure that `wantodance_music_path`, `wantodance_keyframes`, and `wantodance_keyframes_mask` are properly split accordingly.
+# *   The audio file specified by `wantodance_music_path` must match the video duration, calculated as (`num_frames` / 30) seconds.
+# *   The width and height of `wantodance_reference_image` must be multiples of 16.
+# *   `wantodance_fps` is configurable, but since the model appears to have been trained exclusively at 30 FPS, setting it to other values is not recommended.
+# *   In `wantodance_keyframes`, frames that are not keyframes should be solid black.
+# *   `wantodance_keyframes_mask` indicates the positions of valid frames within `wantodance_keyframes`.
+wantodance_keyframes = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Dancer-14B-local/keyframes.mp4")
+wantodance_keyframes = [wantodance_keyframes[i] for i in range(149)]
+video = pipe(
+    prompt="一个人正在跳舞，舞蹈种类是古典舞,图像清晰程度高,人物动作平均幅度中等,人物动作最大幅度中等。, 帧率是30fps。",
+    negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走",
+    seed=0, tiled=True,
+    height=1280, width=720, num_frames=149,
+    num_inference_steps=24,
+    wantodance_music_path="data/diffsynth_example_dataset/wanvideo/Wan-Dancer-14B-local/music.wav",
+    wantodance_reference_image=Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Dancer-14B-local/refimage.jpg"),
+    wantodance_fps=30,
+    wantodance_keyframes=wantodance_keyframes,
+    wantodance_keyframes_mask=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                               1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                               1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                               1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                               1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                               1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                               1],
+)
+save_video(video, "video_Wan-Dancer-14B-local.mp4", fps=30, quality=5)

examples/wanvideo/model_inference/Wan2.1-1.3b-speedcontrol-v1.py

@@ -0,0 +1,34 @@
+import torch
+from PIL import Image
+from diffsynth.utils.data import save_video, VideoData
+from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig
+
+
+pipe = WanVideoPipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="diffusion_pytorch_model*.safetensors"),
+        ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth"),
+        ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="Wan2.1_VAE.pth"),
+        ModelConfig(model_id="DiffSynth-Studio/Wan2.1-1.3b-speedcontrol-v1", origin_file_pattern="model.safetensors"),
+    ],
+    tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="google/umt5-xxl/"),
+)
+
+# Text-to-video
+video = pipe(
+    prompt="纪实摄影风格画面，一只活泼的小狗在绿茵茵的草地上迅速奔跑。小狗毛色棕黄，两只耳朵立起，神情专注而欢快。阳光洒在它身上，使得毛发看上去格外柔软而闪亮。背景是一片开阔的草地，偶尔点缀着几朵野花，远处隐约可见蓝天和几片白云。透视感鲜明，捕捉小狗奔跑时的动感和四周草地的生机。中景侧面移动视角。",
+    negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走",
+    seed=1, tiled=True,
+    motion_bucket_id=0
+)
+save_video(video, "video_slow_Wan2.1-1.3b-speedcontrol-v1.mp4", fps=15, quality=5)
+
+video = pipe(
+    prompt="纪实摄影风格画面，一只活泼的小狗在绿茵茵的草地上迅速奔跑。小狗毛色棕黄，两只耳朵立起，神情专注而欢快。阳光洒在它身上，使得毛发看上去格外柔软而闪亮。背景是一片开阔的草地，偶尔点缀着几朵野花，远处隐约可见蓝天和几片白云。透视感鲜明，捕捉小狗奔跑时的动感和四周草地的生机。中景侧面移动视角。",
+    negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走",
+    seed=1, tiled=True,
+    motion_bucket_id=100
+)
+save_video(video, "video_fast_Wan2.1-1.3b-speedcontrol-v1.mp4", fps=15, quality=5)

examples/wanvideo/model_inference/Wan2.1-FLF2V-14B-720P.py

@@ -0,0 +1,36 @@
+import torch
+from PIL import Image
+from diffsynth.utils.data import save_video, VideoData
+from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig
+from modelscope import dataset_snapshot_download
+
+
+pipe = WanVideoPipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="Wan-AI/Wan2.1-FLF2V-14B-720P", origin_file_pattern="diffusion_pytorch_model*.safetensors"),
+        ModelConfig(model_id="Wan-AI/Wan2.1-FLF2V-14B-720P", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth"),
+        ModelConfig(model_id="Wan-AI/Wan2.1-FLF2V-14B-720P", origin_file_pattern="Wan2.1_VAE.pth"),
+        ModelConfig(model_id="Wan-AI/Wan2.1-FLF2V-14B-720P", origin_file_pattern="models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"),
+    ],
+    tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="google/umt5-xxl/"),
+)
+
+dataset_snapshot_download(
+    dataset_id="DiffSynth-Studio/examples_in_diffsynth",
+    local_dir="./",
+    allow_file_pattern=["data/examples/wan/first_frame.jpeg", "data/examples/wan/last_frame.jpeg"]
+)
+
+# First and last frame to video
+video = pipe(
+    prompt="写实风格，一个女生手持枯萎的花站在花园中，镜头逐渐拉远，记录下花园的全貌。",
+    negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走",
+    input_image=Image.open("data/examples/wan/first_frame.jpeg").resize((960, 960)),
+    end_image=Image.open("data/examples/wan/last_frame.jpeg").resize((960, 960)),
+    seed=0, tiled=True,
+    height=960, width=960, num_frames=33,
+    sigma_shift=16,
+)
+save_video(video, "video_Wan2.1-FLF2V-14B-720P.mp4", fps=15, quality=5)

examples/wanvideo/model_inference/Wan2.1-Fun-1.3B-Control.py

@@ -0,0 +1,34 @@
+import torch
+from PIL import Image
+from diffsynth.utils.data import save_video, VideoData
+from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig
+from modelscope import dataset_snapshot_download
+
+
+pipe = WanVideoPipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="PAI/Wan2.1-Fun-1.3B-Control", origin_file_pattern="diffusion_pytorch_model*.safetensors"),
+        ModelConfig(model_id="PAI/Wan2.1-Fun-1.3B-Control", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth"),
+        ModelConfig(model_id="PAI/Wan2.1-Fun-1.3B-Control", origin_file_pattern="Wan2.1_VAE.pth"),
+        ModelConfig(model_id="PAI/Wan2.1-Fun-1.3B-Control", origin_file_pattern="models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"),
+    ],
+    tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="google/umt5-xxl/"),
+)
+
+dataset_snapshot_download(
+    dataset_id="DiffSynth-Studio/examples_in_diffsynth",
+    local_dir="./",
+    allow_file_pattern=f"data/examples/wan/control_video.mp4"
+)
+
+# Control video
+control_video = VideoData("data/examples/wan/control_video.mp4", height=832, width=576)
+video = pipe(
+    prompt="扁平风格动漫，一位长发少女优雅起舞。她五官精致，大眼睛明亮有神，黑色长发柔顺光泽。身穿淡蓝色T恤和深蓝色牛仔短裤。背景是粉色。",
+    negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走",
+    control_video=control_video, height=832, width=576, num_frames=49,
+    seed=1, tiled=True
+)
+save_video(video, "video_Wan2.1-Fun-1.3B-Control.mp4", fps=15, quality=5)

examples/wanvideo/model_inference/Wan2.1-Fun-1.3B-InP.py

@@ -0,0 +1,36 @@
+import torch
+from PIL import Image
+from diffsynth.utils.data import save_video, VideoData
+from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig
+from modelscope import dataset_snapshot_download
+
+
+pipe = WanVideoPipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="PAI/Wan2.1-Fun-1.3B-InP", origin_file_pattern="diffusion_pytorch_model*.safetensors"),
+        ModelConfig(model_id="PAI/Wan2.1-Fun-1.3B-InP", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth"),
+        ModelConfig(model_id="PAI/Wan2.1-Fun-1.3B-InP", origin_file_pattern="Wan2.1_VAE.pth"),
+        ModelConfig(model_id="PAI/Wan2.1-Fun-1.3B-InP", origin_file_pattern="models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"),
+    ],
+    tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="google/umt5-xxl/"),
+)
+
+dataset_snapshot_download(
+    dataset_id="DiffSynth-Studio/examples_in_diffsynth",
+    local_dir="./",
+    allow_file_pattern=f"data/examples/wan/input_image.jpg"
+)
+image = Image.open("data/examples/wan/input_image.jpg")
+
+# First and last frame to video
+video = pipe(
+    prompt="一艘小船正勇敢地乘风破浪前行。蔚蓝的大海波涛汹涌，白色的浪花拍打着船身，但小船毫不畏惧，坚定地驶向远方。阳光洒在水面上，闪烁着金色的光芒，为这壮丽的场景增添了一抹温暖。镜头拉近，可以看到船上的旗帜迎风飘扬，象征着不屈的精神与冒险的勇气。这段画面充满力量，激励人心，展现了面对挑战时的无畏与执着。",
+    negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走",
+    input_image=image,
+    seed=0, tiled=True
+    # You can input `end_image=xxx` to control the last frame of the video.
+    # The model will automatically generate the dynamic content between `input_image` and `end_image`.
+)
+save_video(video, "video_Wan2.1-Fun-1.3B-InP.mp4", fps=15, quality=5)

examples/wanvideo/model_inference/Wan2.1-Fun-14B-Control.py

@@ -0,0 +1,34 @@
+import torch
+from PIL import Image
+from diffsynth.utils.data import save_video, VideoData
+from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig
+from modelscope import dataset_snapshot_download
+
+
+pipe = WanVideoPipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="PAI/Wan2.1-Fun-14B-Control", origin_file_pattern="diffusion_pytorch_model*.safetensors"),
+        ModelConfig(model_id="PAI/Wan2.1-Fun-14B-Control", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth"),
+        ModelConfig(model_id="PAI/Wan2.1-Fun-14B-Control", origin_file_pattern="Wan2.1_VAE.pth"),
+        ModelConfig(model_id="PAI/Wan2.1-Fun-14B-Control", origin_file_pattern="models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"),
+    ],
+    tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="google/umt5-xxl/"),
+)
+
+dataset_snapshot_download(
+    dataset_id="DiffSynth-Studio/examples_in_diffsynth",
+    local_dir="./",
+    allow_file_pattern=f"data/examples/wan/control_video.mp4"
+)
+
+# Control video
+control_video = VideoData("data/examples/wan/control_video.mp4", height=832, width=576)
+video = pipe(
+    prompt="扁平风格动漫，一位长发少女优雅起舞。她五官精致，大眼睛明亮有神，黑色长发柔顺光泽。身穿淡蓝色T恤和深蓝色牛仔短裤。背景是粉色。",
+    negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走",
+    control_video=control_video, height=832, width=576, num_frames=49,
+    seed=1, tiled=True
+)
+save_video(video, "video_Wan2.1-Fun-14B-Control.mp4", fps=15, quality=5)

examples/wanvideo/model_inference/Wan2.1-Fun-14B-InP.py

@@ -0,0 +1,36 @@
+import torch
+from PIL import Image
+from diffsynth.utils.data import save_video, VideoData
+from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig
+from modelscope import dataset_snapshot_download
+
+
+pipe = WanVideoPipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="PAI/Wan2.1-Fun-14B-InP", origin_file_pattern="diffusion_pytorch_model*.safetensors"),
+        ModelConfig(model_id="PAI/Wan2.1-Fun-14B-InP", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth"),
+        ModelConfig(model_id="PAI/Wan2.1-Fun-14B-InP", origin_file_pattern="Wan2.1_VAE.pth"),
+        ModelConfig(model_id="PAI/Wan2.1-Fun-14B-InP", origin_file_pattern="models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"),
+    ],
+    tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="google/umt5-xxl/"),
+)
+
+dataset_snapshot_download(
+    dataset_id="DiffSynth-Studio/examples_in_diffsynth",
+    local_dir="./",
+    allow_file_pattern=f"data/examples/wan/input_image.jpg"
+)
+image = Image.open("data/examples/wan/input_image.jpg")
+
+# First and last frame to video
+video = pipe(
+    prompt="一艘小船正勇敢地乘风破浪前行。蔚蓝的大海波涛汹涌，白色的浪花拍打着船身，但小船毫不畏惧，坚定地驶向远方。阳光洒在水面上，闪烁着金色的光芒，为这壮丽的场景增添了一抹温暖。镜头拉近，可以看到船上的旗帜迎风飘扬，象征着不屈的精神与冒险的勇气。这段画面充满力量，激励人心，展现了面对挑战时的无畏与执着。",
+    negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走",
+    input_image=image,
+    seed=0, tiled=True
+    # You can input `end_image=xxx` to control the last frame of the video.
+    # The model will automatically generate the dynamic content between `input_image` and `end_image`.
+)
+save_video(video, "video_Wan2.1-Fun-14B-InP.mp4", fps=15, quality=5)

examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py

@@ -0,0 +1,44 @@
+import torch
+from PIL import Image
+from diffsynth.utils.data import save_video, VideoData
+from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig
+from modelscope import dataset_snapshot_download
+
+
+pipe = WanVideoPipeline.from_pretrained(
+    torch_dtype=torch.bfloat16,
+    device="cuda",
+    model_configs=[
+        ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera", origin_file_pattern="diffusion_pytorch_model*.safetensors"),
+        ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth"),
+        ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera", origin_file_pattern="Wan2.1_VAE.pth"),
+        ModelConfig(model_id="PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera", origin_file_pattern="models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"),
+    ],
+    tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="google/umt5-xxl/"),
+)
+
+
+dataset_snapshot_download(
+    dataset_id="DiffSynth-Studio/examples_in_diffsynth",
+    local_dir="./",
+    allow_file_pattern=f"data/examples/wan/input_image.jpg"
+)
+input_image = Image.open("data/examples/wan/input_image.jpg")
+
+video = pipe(
+    prompt="一艘小船正勇敢地乘风破浪前行。蔚蓝的大海波涛汹涌，白色的浪花拍打着船身，但小船毫不畏惧，坚定地驶向远方。阳光洒在水面上，闪烁着金色的光芒，为这壮丽的场景增添了一抹温暖。镜头拉近，可以看到船上的旗帜迎风飘扬，象征着不屈的精神与冒险的勇气。这段画面充满力量，激励人心，展现了面对挑战时的无畏与执着。",
+    negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走",
+    seed=0, tiled=True,
+    input_image=input_image,
+    camera_control_direction="Left", camera_control_speed=0.01,
+)
+save_video(video, "video_left_Wan2.1-Fun-V1.1-1.3B-Control-Camera.mp4", fps=15, quality=5)
+
+video = pipe(
+    prompt="一艘小船正勇敢地乘风破浪前行。蔚蓝的大海波涛汹涌，白色的浪花拍打着船身，但小船毫不畏惧，坚定地驶向远方。阳光洒在水面上，闪烁着金色的光芒，为这壮丽的场景增添了一抹温暖。镜头拉近，可以看到船上的旗帜迎风飘扬，象征着不屈的精神与冒险的勇气。这段画面充满力量，激励人心，展现了面对挑战时的无畏与执着。",
+    negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走",
+    seed=0, tiled=True,
+    input_image=input_image,
+    camera_control_direction="Up", camera_control_speed=0.01,
+)
+save_video(video, "video_up_Wan2.1-Fun-V1.1-1.3B-Control-Camera.mp4", fps=15, quality=5)