dagloop5 commited on
Commit
ace7d16
·
verified ·
1 Parent(s): 60b324f

Update h3_momentum.py

Browse files
Files changed (1) hide show
  1. h3_momentum.py +12 -5
h3_momentum.py CHANGED
@@ -31,6 +31,7 @@ from diffusers.modular_pipelines.minimax_h3.before_denoise import (
31
  MiniMaxH3SetTimestepsStep,
32
  )
33
  from diffusers.modular_pipelines.minimax_h3.decoders import MiniMaxH3AfterDenoiseStep
 
34
  from diffusers.modular_pipelines.minimax_h3.denoise import (
35
  MiniMaxH3DenoiseLoopWrapper,
36
  MiniMaxH3LoopDenoiser,
@@ -133,11 +134,17 @@ class MiniMaxH3MomentumConditionStep(ModularPipelineBlocks):
133
  channels = components.vae_latent_channels
134
  patch_size = components.patch_size
135
 
136
- # (num_frames, 3, H, W) in [0, 1] -> (1, 3, num_frames, H, W), the video VAE's own input convention.
137
- frames = block_state.given_video.to(device=device, dtype=components.vae.dtype)
138
- frames = frames.permute(1, 0, 2, 3)[None]
139
- posterior = components.vae.encode(frames, return_dict=False)[0]
140
- momentum_latents = posterior.mode() # (1, channels, k, latent_height, latent_width)
 
 
 
 
 
 
141
 
142
  given_rows = pack_video_rows(momentum_latents, patch_size, channels).to(block_state.latents.dtype)
143
  num_momentum_rows = given_rows.shape[0]
 
31
  MiniMaxH3SetTimestepsStep,
32
  )
33
  from diffusers.modular_pipelines.minimax_h3.decoders import MiniMaxH3AfterDenoiseStep
34
+ from diffusers.modular_pipelines.minimax_h3.encoders import encode_vae_condition
35
  from diffusers.modular_pipelines.minimax_h3.denoise import (
36
  MiniMaxH3DenoiseLoopWrapper,
37
  MiniMaxH3LoopDenoiser,
 
134
  channels = components.vae_latent_channels
135
  patch_size = components.patch_size
136
 
137
+ # (num_frames, 3, H, W) uint8 -> (1, 3, num_frames, H, W), `encode_vae_condition`'s own documented
138
+ # input convention — the same helper `MiniMaxH3KeyframeVaeEncoderStep` uses for a keyframe, so the
139
+ # ImageNet pixel normalization, the seeded posterior *sample* (not `.mode()`), the float16 rounding,
140
+ # and the final latents_mean/latents_std normalization are all exactly what the released model was
141
+ # actually conditioned on. Reproducing this by hand was the bug in the first version of this block —
142
+ # not the row-packing math, which stays as derived.
143
+ pixels = block_state.given_video.to(device=device).permute(1, 0, 2, 3)[None]
144
+ momentum_latents = encode_vae_condition(
145
+ components.vae, pixels, components.pixel_mean, components.pixel_std,
146
+ components.keyframe_encode_seed,
147
+ ).to(device=device) # (1, channels, k, latent_height, latent_width)
148
 
149
  given_rows = pack_video_rows(momentum_latents, patch_size, channels).to(block_state.latents.dtype)
150
  num_momentum_rows = given_rows.shape[0]