Hugging Face's logo

TeleEmbodied
/
WNM-3D

Robotics
Safetensors
PyTorch
vision-language-navigation
embodied-ai
world-model
video-generation
Model card Files
xet
 Community
Leeyingke commited on
Commit
c0deede
·
verified ·
1 Parent(s): 696b30a

Add files using upload-large-folder tool

Browse files
Files changed (33) hide show
  1. wnm_3d_stage1/action_normalization.json +42 -0
  2. wnm_3d_stage1/config.json +93 -0
  3. wnm_3d_stage1/experiment_cfg/conf.yaml +582 -0
  4. wnm_3d_stage1/experiment_cfg/metadata.json +107 -0
  5. wnm_3d_stage1/model-00001-of-00006.safetensors +3 -0
  6. wnm_3d_stage1/model-00002-of-00006.safetensors +3 -0
  7. wnm_3d_stage1/model-00003-of-00006.safetensors +3 -0
  8. wnm_3d_stage1/model-00004-of-00006.safetensors +3 -0
  9. wnm_3d_stage1/model-00005-of-00006.safetensors +3 -0
  10. wnm_3d_stage1/model-00006-of-00006.safetensors +3 -0
  11. wnm_3d_stage1/model.safetensors.index.json +0 -0
  12. wnm_3d_stage2/action_normalization.json +42 -0
  13. wnm_3d_stage2/config.json +93 -0
  14. wnm_3d_stage2/experiment_cfg/conf.yaml +581 -0
  15. wnm_3d_stage2/experiment_cfg/metadata.json +107 -0
  16. wnm_3d_stage2/model-00001-of-00006.safetensors +3 -0
  17. wnm_3d_stage2/model-00002-of-00006.safetensors +3 -0
  18. wnm_3d_stage2/model-00003-of-00006.safetensors +3 -0
  19. wnm_3d_stage2/model-00004-of-00006.safetensors +3 -0
  20. wnm_3d_stage2/model-00005-of-00006.safetensors +3 -0
  21. wnm_3d_stage2/model-00006-of-00006.safetensors +3 -0
  22. wnm_3d_stage2/model.safetensors.index.json +0 -0
  23. wnm_3d_stage3/action_normalization.json +42 -0
  24. wnm_3d_stage3/config.json +93 -0
  25. wnm_3d_stage3/experiment_cfg/conf.yaml +581 -0
  26. wnm_3d_stage3/experiment_cfg/metadata.json +107 -0
  27. wnm_3d_stage3/model-00001-of-00006.safetensors +3 -0
  28. wnm_3d_stage3/model-00002-of-00006.safetensors +3 -0
  29. wnm_3d_stage3/model-00003-of-00006.safetensors +3 -0
  30. wnm_3d_stage3/model-00004-of-00006.safetensors +3 -0
  31. wnm_3d_stage3/model-00005-of-00006.safetensors +3 -0
  32. wnm_3d_stage3/model-00006-of-00006.safetensors +3 -0
  33. wnm_3d_stage3/model.safetensors.index.json +0 -0
wnm_3d_stage1/action_normalization.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema": "wnm_3d_action_normalization_v1",
3
+ "embodiment": "interiorgs",
4
+ "action_key": "action.nav_delta",
5
+ "channel_order": [
6
+ "dx_m",
7
+ "dy_m",
8
+ "dyaw_rad"
9
+ ],
10
+ "model_action_dim": 32,
11
+ "action_horizon_per_chunk": 8,
12
+ "decoded_action_dims": 3,
13
+ "normalization_mode": "q99",
14
+ "normalized_reference_range": [
15
+ -1.0,
16
+ 1.0
17
+ ],
18
+ "q01": [
19
+ -0.5167848467826843,
20
+ -0.6000025272369385,
21
+ -0.7039973735809326
22
+ ],
23
+ "q99": [
24
+ 0.8178253173828125,
25
+ 0.6459061503410339,
26
+ 0.7619411945343018
27
+ ],
28
+ "nav_action_scale": 4.0,
29
+ "physical_q01_after_scale": [
30
+ -0.12919621169567108,
31
+ -0.15000063180923462,
32
+ -0.17599934339523315
33
+ ],
34
+ "physical_q99_after_scale": [
35
+ 0.20445632934570312,
36
+ 0.16147653758525848,
37
+ 0.19048529863357544
38
+ ],
39
+ "decode_formula": "delta = (((action_norm[..., :3] + 1) * 0.5) * (q99 - q01) + q01) / nav_action_scale",
40
+ "encode_formula": "action_norm = 2 * ((delta * nav_action_scale) - q01) / (q99 - q01) - 1",
41
+ "decode_clamps_normalized_action": false
42
+ }
wnm_3d_stage1/config.json ADDED
@@ -0,0 +1,93 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_dim": 32,
3
+ "action_head_cfg": {
4
+ "_convert_": "object",
5
+ "_target_": "gammanav.vln.model.wnm_3d.action_head.wan_flow_matching_action_tf.WANPolicyHead",
6
+ "config": {
7
+ "_recursive_": false,
8
+ "_target_": "gammanav.vln.model.wnm_3d.action_head.wan_flow_matching_action_tf.WANPolicyHeadConfig",
9
+ "action_dim": 32,
10
+ "action_horizon": 8,
11
+ "backbone_features_projector_cfg": null,
12
+ "decouple_video_action_noise": false,
13
+ "diffusion_model_cfg": {
14
+ "_convert_": "object",
15
+ "_target_": "gammanav.vln.model.wnm_3d.modules.wan_video_dit_action_casual_chunk.CausalWanModel",
16
+ "concat_first_frame_latent": false,
17
+ "diffusion_model_pretrained_path": "./checkpoints/Wan2.2-TI2V-5B",
18
+ "dim": 3072,
19
+ "eps": 1e-06,
20
+ "ffn_dim": 14336,
21
+ "frame_seqlen": 50,
22
+ "freq_dim": 256,
23
+ "in_dim": 48,
24
+ "max_chunk_size": 4,
25
+ "model_type": "ti2v",
26
+ "num_action_per_block": 8,
27
+ "num_frame_per_block": 2,
28
+ "num_heads": 24,
29
+ "num_layers": 30,
30
+ "num_state_per_block": 1,
31
+ "out_dim": 48
32
+ },
33
+ "image_encoder_cfg": {
34
+ "_convert_": "object",
35
+ "_target_": "gammanav.vln.model.wnm_3d.modules.wan_video_image_encoder.WanImageEncoder",
36
+ "image_encoder_pretrained_path": "./checkpoints/Wan2.1-I2V-14B-480P/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"
37
+ },
38
+ "init_lora_weights": "kaiming",
39
+ "lora_alpha": 4,
40
+ "lora_rank": 4,
41
+ "lora_target_modules": "q,k,v,o,ffn.0,ffn.2",
42
+ "max_action_dim": 32,
43
+ "max_state_dim": 64,
44
+ "num_frame_per_block": 2,
45
+ "num_frames": 33,
46
+ "num_inference_timesteps": 4,
47
+ "skip_component_loading": true,
48
+ "target_video_height": 160,
49
+ "target_video_width": 320,
50
+ "text_encoder_cfg": {
51
+ "_convert_": "object",
52
+ "_target_": "gammanav.vln.model.wnm_3d.modules.wan_video_text_encoder.WanTextEncoder",
53
+ "text_encoder_pretrained_path": "./checkpoints/Wan2.2-TI2V-5B/models_t5_umt5-xxl-enc-bf16.pth"
54
+ },
55
+ "tile_size_height": 34,
56
+ "tile_size_width": 34,
57
+ "tile_stride_height": 18,
58
+ "tile_stride_width": 16,
59
+ "tiled": false,
60
+ "train_architecture": "full",
61
+ "tune_diffusion_model": true,
62
+ "tune_projector": true,
63
+ "use_vggt_geometry_adapter": true,
64
+ "vae_cfg": {
65
+ "_convert_": "object",
66
+ "_target_": "gammanav.vln.model.wnm_3d.modules.wan_video_vae.WanVideoVAE38",
67
+ "dim": 160,
68
+ "vae_pretrained_path": "./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth",
69
+ "z_dim": 48
70
+ },
71
+ "vggt_adapter_blocks": 2,
72
+ "vggt_adapter_dim": 512,
73
+ "vggt_adapter_heads": 8,
74
+ "vggt_checkpoint_path": "./checkpoints/vggt_omega_1b_512.pt",
75
+ "vggt_image_resolution": 512,
76
+ "vggt_resize_mode": "square",
77
+ "video_noise_beta_alpha": 3.0,
78
+ "video_noise_beta_beta": 1.0
79
+ }
80
+ },
81
+ "action_horizon": 8,
82
+ "architectures": [
83
+ "WNM3DModel"
84
+ ],
85
+ "backbone_cfg": {
86
+ "_target_": "gammanav.vln.model.wnm_3d.backbone.identity.IdentityBackbone"
87
+ },
88
+ "hidden_size": 0,
89
+ "model_dtype": "float32",
90
+ "model_type": "wnm_3d",
91
+ "torch_dtype": "bfloat16",
92
+ "transformers_version": "4.51.3"
93
+ }
wnm_3d_stage1/experiment_cfg/conf.yaml ADDED
@@ -0,0 +1,582 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ _target_: gammanav.vln.model.wnm_3d.model.WNM3DModel
3
+ _convert_: object
4
+ config:
5
+ _target_: gammanav.vln.model.wnm_3d.model.WNM3DConfig
6
+ _recursive_: false
7
+ model_dtype: float32
8
+ hidden_size: 0
9
+ action_horizon: 8
10
+ action_dim: 32
11
+ backbone_cfg:
12
+ _target_: gammanav.vln.model.wnm_3d.backbone.identity.IdentityBackbone
13
+ action_head_cfg:
14
+ config:
15
+ backbone_features_projector_cfg: null
16
+ _target_: gammanav.vln.model.wnm_3d.action_head.wan_flow_matching_action_tf.WANPolicyHeadConfig
17
+ _recursive_: false
18
+ tiled: false
19
+ tile_size_height: 34
20
+ tile_size_width: 34
21
+ tile_stride_height: 18
22
+ tile_stride_width: 16
23
+ lora_rank: 4
24
+ lora_alpha: 4
25
+ num_frames: 33
26
+ num_frame_per_block: 2
27
+ lora_target_modules: q,k,v,o,ffn.0,ffn.2
28
+ init_lora_weights: kaiming
29
+ train_architecture: full
30
+ use_vggt_geometry_adapter: true
31
+ vggt_checkpoint_path: ./checkpoints/vggt_omega_1b_512.pt
32
+ vggt_image_resolution: 512
33
+ vggt_resize_mode: square
34
+ vggt_adapter_dim: 512
35
+ vggt_adapter_blocks: 2
36
+ vggt_adapter_heads: 8
37
+ max_state_dim: 64
38
+ max_action_dim: 32
39
+ diffusion_model_cfg:
40
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_dit_action_casual_chunk.CausalWanModel
41
+ _convert_: object
42
+ diffusion_model_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B
43
+ model_type: ti2v
44
+ frame_seqlen: 50
45
+ dim: 3072
46
+ in_dim: 48
47
+ ffn_dim: 14336
48
+ out_dim: 48
49
+ freq_dim: 256
50
+ eps: 1.0e-06
51
+ num_heads: 24
52
+ num_layers: 30
53
+ max_chunk_size: 4
54
+ num_frame_per_block: 2
55
+ num_action_per_block: 8
56
+ num_state_per_block: 1
57
+ concat_first_frame_latent: false
58
+ text_encoder_cfg:
59
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_text_encoder.WanTextEncoder
60
+ _convert_: object
61
+ text_encoder_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B/models_t5_umt5-xxl-enc-bf16.pth
62
+ image_encoder_cfg:
63
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_image_encoder.WanImageEncoder
64
+ _convert_: object
65
+ image_encoder_pretrained_path: ./checkpoints/Wan2.1-I2V-14B-480P/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth
66
+ vae_cfg:
67
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_vae.WanVideoVAE38
68
+ _convert_: object
69
+ vae_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth
70
+ z_dim: 48
71
+ dim: 160
72
+ action_dim: 32
73
+ action_horizon: 8
74
+ num_inference_timesteps: 4
75
+ decouple_video_action_noise: false
76
+ video_noise_beta_alpha: 3.0
77
+ video_noise_beta_beta: 1.0
78
+ tune_projector: true
79
+ tune_diffusion_model: true
80
+ target_video_height: 160
81
+ target_video_width: 320
82
+ _target_: gammanav.vln.model.wnm_3d.action_head.wan_flow_matching_action_tf.WANPolicyHead
83
+ _convert_: object
84
+ train_dataset:
85
+ _target_: gammanav.vln.data.dataset.lerobot.LeRobotMixtureDataset.from_mixture_spec
86
+ _convert_: object
87
+ mixture_spec:
88
+ - dataset_path:
89
+ interiorgs:
90
+ - ./data/interiorgs_lerobot_seen
91
+ dataset_weight: 1.0
92
+ distribute_weights: true
93
+ dataset_class: gammanav.vln.data.dataset.lerobot_sharded.ShardedLeRobotSingleActionChunkDatasetInteriorGS
94
+ all_modality_configs:
95
+ interiorgs:
96
+ video:
97
+ _target_: gammanav.vln.data.dataset.ModalityConfig
98
+ delta_indices: []
99
+ eval_delta_indices:
100
+ - 0
101
+ modality_keys:
102
+ - video.rgb
103
+ state:
104
+ _target_: gammanav.vln.data.dataset.ModalityConfig
105
+ delta_indices: []
106
+ modality_keys:
107
+ - state.nav_pose
108
+ action:
109
+ _target_: gammanav.vln.data.dataset.ModalityConfig
110
+ delta_indices: []
111
+ modality_keys:
112
+ - action.nav_delta
113
+ language:
114
+ _target_: gammanav.vln.data.dataset.ModalityConfig
115
+ delta_indices:
116
+ - 0
117
+ modality_keys:
118
+ - annotation.language.language_instruction
119
+ all_transforms:
120
+ interiorgs:
121
+ _target_: gammanav.vln.data.transform.ComposedModalityTransform
122
+ transforms:
123
+ - _target_: gammanav.vln.data.transform.VideoToTensor
124
+ apply_to:
125
+ - video.rgb
126
+ - _target_: gammanav.vln.data.transform.VideoCrop
127
+ apply_to:
128
+ - video.rgb
129
+ scale: 0.95
130
+ mode: random
131
+ - _target_: gammanav.vln.data.transform.VideoResize
132
+ apply_to:
133
+ - video.rgb
134
+ height: 160
135
+ width: 320
136
+ interpolation: linear
137
+ - _target_: gammanav.vln.data.transform.VideoColorJitter
138
+ apply_to:
139
+ - video.rgb
140
+ brightness: 0.3
141
+ contrast: 0.4
142
+ saturation: 0.5
143
+ hue: 0.08
144
+ - _target_: gammanav.vln.data.transform.VideoToNumpy
145
+ apply_to:
146
+ - video.rgb
147
+ - _target_: gammanav.vln.data.transform.StateActionToTensor
148
+ apply_to:
149
+ - state.nav_pose
150
+ - _target_: gammanav.vln.data.transform.StateActionTransform
151
+ apply_to:
152
+ - state.nav_pose
153
+ normalization_modes:
154
+ state.nav_pose: q99
155
+ - _target_: gammanav.vln.data.transform.StateActionToTensor
156
+ apply_to:
157
+ - action.nav_delta
158
+ - _target_: gammanav.vln.data.transform.StateActionTransform
159
+ apply_to:
160
+ - action.nav_delta
161
+ normalization_modes:
162
+ action.nav_delta: q99
163
+ - _target_: gammanav.vln.data.transform.ConcatTransform
164
+ video_concat_order:
165
+ - video.rgb
166
+ state_concat_order:
167
+ - state.nav_pose
168
+ action_concat_order:
169
+ - action.nav_delta
170
+ - _target_: gammanav.vln.model.wnm_3d.transform.wnm_3d_cotrain.WNM3DTransform
171
+ default_instruction: Perform the default behavior.
172
+ language_dropout_prob: 0.0
173
+ always_use_default_instruction: false
174
+ max_state_dim: 64
175
+ max_action_dim: 32
176
+ max_length: 512
177
+ state_horizon: 4
178
+ action_horizon: 8
179
+ tokenizer_path: ./checkpoints/umt5-xxl
180
+ dataset_kwargs:
181
+ max_chunk_size: 4
182
+ num_frames: 33
183
+ action_horizon: 8
184
+ state_horizon: 4
185
+ num_frame_per_block: 2
186
+ num_action_per_block: 8
187
+ num_state_per_block: 1
188
+ vae_temporal_stride: 4
189
+ video_frame_stride: 1
190
+ action_frame_stride: 1
191
+ history_sampling: uniform
192
+ nav_action_scale: 4.0
193
+ nav_smooth_window: 7
194
+ nav_smooth_passes: 2
195
+ compute_statistics: true
196
+ max_stats_episodes: 512
197
+ stats_start_stride: 16
198
+ mixture_kwargs:
199
+ training: true
200
+ balance_dataset_weights: false
201
+ balance_trajectory_weights: true
202
+ seed: 42
203
+ allow_padding_at_end: true
204
+ trainer:
205
+ _target_: gammanav.vln.experiment.VLNTrainer
206
+ _partial_: true
207
+ _recursive_: false
208
+ callbacks: null
209
+ model: ???
210
+ train_dataset: ???
211
+ compute_dtype: ???
212
+ wandb_project: wnm_3d
213
+ output_dir: ./checkpoints/wnm_3d_stage1
214
+ load_from_yaml: null
215
+ remove_unused_columns: false
216
+ bf16: true
217
+ tf32: true
218
+ global_batch_size: 8
219
+ raise_error_if_global_batch_size_not_set: false
220
+ per_device_train_batch_size: 1
221
+ per_device_eval_batch_size: 64
222
+ gradient_accumulation_steps: 1
223
+ dataloader_num_workers: 0
224
+ dataloader_pin_memory: false
225
+ dataloader_persistent_workers: false
226
+ optim: adamw_torch
227
+ learning_rate: 0.0001
228
+ adam_beta1: 0.95
229
+ adam_beta2: 0.999
230
+ adam_epsilon: 1.0e-08
231
+ weight_decay: 1.0e-05
232
+ lr_scheduler_type: cosine
233
+ warmup_ratio: 0.05
234
+ logging_steps: 1
235
+ num_train_epochs: 1000
236
+ max_steps: 10
237
+ save_strategy: steps
238
+ save_steps: 10
239
+ eval_strategy: 'no'
240
+ save_total_limit: 5
241
+ report_to: none
242
+ seed: 42
243
+ do_eval: false
244
+ gradient_checkpointing: false
245
+ ddp_find_unused_parameters: false
246
+ ddp_bucket_cap_mb: 100
247
+ eval_bf16: true
248
+ torch_compile_mode: null
249
+ pretrained_model_path: null
250
+ save_lora_only: false
251
+ training_args:
252
+ _target_: transformers.TrainingArguments
253
+ output_dir: ./checkpoints/wnm_3d_stage1
254
+ run_name: wnm_3d_10step_reference
255
+ remove_unused_columns: false
256
+ deepspeed: gammanav/vln/configs/deepspeed/zero2.json
257
+ gradient_checkpointing: false
258
+ bf16: true
259
+ tf32: true
260
+ per_device_train_batch_size: 1
261
+ per_device_eval_batch_size: 64
262
+ gradient_accumulation_steps: 1
263
+ dataloader_num_workers: 0
264
+ dataloader_pin_memory: false
265
+ dataloader_persistent_workers: false
266
+ optim: adamw_torch
267
+ adam_beta1: 0.95
268
+ adam_beta2: 0.999
269
+ adam_epsilon: 1.0e-08
270
+ learning_rate: 1.0e-05
271
+ weight_decay: 1.0e-05
272
+ warmup_ratio: 0.05
273
+ lr_scheduler_type: cosine
274
+ logging_steps: 1
275
+ num_train_epochs: 1000
276
+ max_steps: 10
277
+ save_strategy: steps
278
+ save_steps: 10
279
+ save_total_limit: 5
280
+ report_to: none
281
+ seed: 42
282
+ do_eval: false
283
+ ddp_find_unused_parameters: false
284
+ ddp_bucket_cap_mb: 100
285
+ torch_compile_mode: null
286
+ logging_dir: ./checkpoints/wnm_3d_stage1/tensorboard
287
+ profile_dir: null
288
+ backbone_hidden_size: 0
289
+ backbone_cfg:
290
+ _target_: gammanav.vln.model.wnm_3d.backbone.identity.IdentityBackbone
291
+ action_head_cfg:
292
+ config:
293
+ backbone_features_projector_cfg: null
294
+ _target_: gammanav.vln.model.wnm_3d.action_head.wan_flow_matching_action_tf.WANPolicyHeadConfig
295
+ _recursive_: false
296
+ tiled: false
297
+ tile_size_height: 34
298
+ tile_size_width: 34
299
+ tile_stride_height: 18
300
+ tile_stride_width: 16
301
+ lora_rank: 4
302
+ lora_alpha: 4
303
+ num_frames: 33
304
+ num_frame_per_block: 2
305
+ lora_target_modules: q,k,v,o,ffn.0,ffn.2
306
+ init_lora_weights: kaiming
307
+ train_architecture: full
308
+ use_vggt_geometry_adapter: true
309
+ vggt_checkpoint_path: ./checkpoints/vggt_omega_1b_512.pt
310
+ vggt_image_resolution: 512
311
+ vggt_resize_mode: square
312
+ vggt_adapter_dim: 512
313
+ vggt_adapter_blocks: 2
314
+ vggt_adapter_heads: 8
315
+ max_state_dim: 64
316
+ max_action_dim: 32
317
+ diffusion_model_cfg:
318
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_dit_action_casual_chunk.CausalWanModel
319
+ _convert_: object
320
+ diffusion_model_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B
321
+ model_type: ti2v
322
+ frame_seqlen: 50
323
+ dim: 3072
324
+ in_dim: 48
325
+ ffn_dim: 14336
326
+ out_dim: 48
327
+ freq_dim: 256
328
+ eps: 1.0e-06
329
+ num_heads: 24
330
+ num_layers: 30
331
+ max_chunk_size: 4
332
+ num_frame_per_block: 2
333
+ num_action_per_block: 8
334
+ num_state_per_block: 1
335
+ concat_first_frame_latent: false
336
+ text_encoder_cfg:
337
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_text_encoder.WanTextEncoder
338
+ _convert_: object
339
+ text_encoder_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B/models_t5_umt5-xxl-enc-bf16.pth
340
+ image_encoder_cfg:
341
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_image_encoder.WanImageEncoder
342
+ _convert_: object
343
+ image_encoder_pretrained_path: ./checkpoints/Wan2.1-I2V-14B-480P/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth
344
+ vae_cfg:
345
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_vae.WanVideoVAE38
346
+ _convert_: object
347
+ vae_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth
348
+ z_dim: 48
349
+ dim: 160
350
+ action_dim: 32
351
+ action_horizon: 8
352
+ num_inference_timesteps: 4
353
+ decouple_video_action_noise: false
354
+ video_noise_beta_alpha: 3.0
355
+ video_noise_beta_beta: 1.0
356
+ tune_projector: true
357
+ tune_diffusion_model: true
358
+ target_video_height: 160
359
+ target_video_width: 320
360
+ _target_: gammanav.vln.model.wnm_3d.action_head.wan_flow_matching_action_tf.WANPolicyHead
361
+ _convert_: object
362
+ dit_version: ./checkpoints/Wan2.2-TI2V-5B
363
+ text_encoder_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B/models_t5_umt5-xxl-enc-bf16.pth
364
+ image_encoder_pretrained_path: ./checkpoints/Wan2.1-I2V-14B-480P/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth
365
+ vae_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth
366
+ train_architecture: full
367
+ num_frame_per_block: 2
368
+ num_action_per_block: 8
369
+ num_state_per_block: 1
370
+ frame_seqlen: 50
371
+ use_vggt_geometry_adapter: true
372
+ vggt_omega_checkpoint_path: ./checkpoints/vggt_omega_1b_512.pt
373
+ vggt_image_resolution: 512
374
+ vggt_resize_mode: square
375
+ vggt_adapter_dim: 512
376
+ vggt_adapter_blocks: 2
377
+ vggt_adapter_heads: 8
378
+ max_length: 512
379
+ tokenizer_path: ./checkpoints/umt5-xxl
380
+ data_collator:
381
+ _target_: gammanav.vln.model.wnm_3d.transform.wnm_3d_cotrain.DefaultDataCollator
382
+ tokenizer_path: ./checkpoints/umt5-xxl
383
+ max_length: 512
384
+ num_visual_tokens_per_frame: 16
385
+ max_state_dim: 64
386
+ max_action_dim: 32
387
+ language_dropout_prob: 0.0
388
+ model_specific_transform:
389
+ _target_: gammanav.vln.model.wnm_3d.transform.wnm_3d_cotrain.WNM3DTransform
390
+ default_instruction: Perform the default behavior.
391
+ language_dropout_prob: 0.0
392
+ always_use_default_instruction: false
393
+ max_state_dim: 64
394
+ max_action_dim: 32
395
+ max_length: 512
396
+ state_horizon: 4
397
+ action_horizon: 8
398
+ tokenizer_path: ./checkpoints/umt5-xxl
399
+ image_resolution_width: 320
400
+ image_resolution_height: 160
401
+ num_frames: 33
402
+ state_horizon: 4
403
+ action_horizon: 8
404
+ vae_temporal_stride: 4
405
+ max_chunk_size: 4
406
+ video_frame_stride: 1
407
+ action_frame_stride: 1
408
+ history_sampling: uniform
409
+ interiorgs_data_root: ./data/interiorgs_lerobot_seen
410
+ modality_config_interiorgs:
411
+ video:
412
+ _target_: gammanav.vln.data.dataset.ModalityConfig
413
+ delta_indices: []
414
+ eval_delta_indices:
415
+ - 0
416
+ modality_keys:
417
+ - video.rgb
418
+ state:
419
+ _target_: gammanav.vln.data.dataset.ModalityConfig
420
+ delta_indices: []
421
+ modality_keys:
422
+ - state.nav_pose
423
+ action:
424
+ _target_: gammanav.vln.data.dataset.ModalityConfig
425
+ delta_indices: []
426
+ modality_keys:
427
+ - action.nav_delta
428
+ language:
429
+ _target_: gammanav.vln.data.dataset.ModalityConfig
430
+ delta_indices:
431
+ - 0
432
+ modality_keys:
433
+ - annotation.language.language_instruction
434
+ transform_interiorgs:
435
+ _target_: gammanav.vln.data.transform.ComposedModalityTransform
436
+ transforms:
437
+ - _target_: gammanav.vln.data.transform.VideoToTensor
438
+ apply_to:
439
+ - video.rgb
440
+ - _target_: gammanav.vln.data.transform.VideoCrop
441
+ apply_to:
442
+ - video.rgb
443
+ scale: 0.95
444
+ mode: random
445
+ - _target_: gammanav.vln.data.transform.VideoResize
446
+ apply_to:
447
+ - video.rgb
448
+ height: 160
449
+ width: 320
450
+ interpolation: linear
451
+ - _target_: gammanav.vln.data.transform.VideoColorJitter
452
+ apply_to:
453
+ - video.rgb
454
+ brightness: 0.3
455
+ contrast: 0.4
456
+ saturation: 0.5
457
+ hue: 0.08
458
+ - _target_: gammanav.vln.data.transform.VideoToNumpy
459
+ apply_to:
460
+ - video.rgb
461
+ - _target_: gammanav.vln.data.transform.StateActionToTensor
462
+ apply_to:
463
+ - state.nav_pose
464
+ - _target_: gammanav.vln.data.transform.StateActionTransform
465
+ apply_to:
466
+ - state.nav_pose
467
+ normalization_modes:
468
+ state.nav_pose: q99
469
+ - _target_: gammanav.vln.data.transform.StateActionToTensor
470
+ apply_to:
471
+ - action.nav_delta
472
+ - _target_: gammanav.vln.data.transform.StateActionTransform
473
+ apply_to:
474
+ - action.nav_delta
475
+ normalization_modes:
476
+ action.nav_delta: q99
477
+ - _target_: gammanav.vln.data.transform.ConcatTransform
478
+ video_concat_order:
479
+ - video.rgb
480
+ state_concat_order:
481
+ - state.nav_pose
482
+ action_concat_order:
483
+ - action.nav_delta
484
+ - _target_: gammanav.vln.model.wnm_3d.transform.wnm_3d_cotrain.WNM3DTransform
485
+ default_instruction: Perform the default behavior.
486
+ language_dropout_prob: 0.0
487
+ always_use_default_instruction: false
488
+ max_state_dim: 64
489
+ max_action_dim: 32
490
+ max_length: 512
491
+ state_horizon: 4
492
+ action_horizon: 8
493
+ tokenizer_path: ./checkpoints/umt5-xxl
494
+ modality_configs:
495
+ interiorgs:
496
+ video:
497
+ _target_: gammanav.vln.data.dataset.ModalityConfig
498
+ delta_indices: []
499
+ eval_delta_indices:
500
+ - 0
501
+ modality_keys:
502
+ - video.rgb
503
+ state:
504
+ _target_: gammanav.vln.data.dataset.ModalityConfig
505
+ delta_indices: []
506
+ modality_keys:
507
+ - state.nav_pose
508
+ action:
509
+ _target_: gammanav.vln.data.dataset.ModalityConfig
510
+ delta_indices: []
511
+ modality_keys:
512
+ - action.nav_delta
513
+ language:
514
+ _target_: gammanav.vln.data.dataset.ModalityConfig
515
+ delta_indices:
516
+ - 0
517
+ modality_keys:
518
+ - annotation.language.language_instruction
519
+ transforms:
520
+ interiorgs:
521
+ _target_: gammanav.vln.data.transform.ComposedModalityTransform
522
+ transforms:
523
+ - _target_: gammanav.vln.data.transform.VideoToTensor
524
+ apply_to:
525
+ - video.rgb
526
+ - _target_: gammanav.vln.data.transform.VideoCrop
527
+ apply_to:
528
+ - video.rgb
529
+ scale: 0.95
530
+ mode: random
531
+ - _target_: gammanav.vln.data.transform.VideoResize
532
+ apply_to:
533
+ - video.rgb
534
+ height: 160
535
+ width: 320
536
+ interpolation: linear
537
+ - _target_: gammanav.vln.data.transform.VideoColorJitter
538
+ apply_to:
539
+ - video.rgb
540
+ brightness: 0.3
541
+ contrast: 0.4
542
+ saturation: 0.5
543
+ hue: 0.08
544
+ - _target_: gammanav.vln.data.transform.VideoToNumpy
545
+ apply_to:
546
+ - video.rgb
547
+ - _target_: gammanav.vln.data.transform.StateActionToTensor
548
+ apply_to:
549
+ - state.nav_pose
550
+ - _target_: gammanav.vln.data.transform.StateActionTransform
551
+ apply_to:
552
+ - state.nav_pose
553
+ normalization_modes:
554
+ state.nav_pose: q99
555
+ - _target_: gammanav.vln.data.transform.StateActionToTensor
556
+ apply_to:
557
+ - action.nav_delta
558
+ - _target_: gammanav.vln.data.transform.StateActionTransform
559
+ apply_to:
560
+ - action.nav_delta
561
+ normalization_modes:
562
+ action.nav_delta: q99
563
+ - _target_: gammanav.vln.data.transform.ConcatTransform
564
+ video_concat_order:
565
+ - video.rgb
566
+ state_concat_order:
567
+ - state.nav_pose
568
+ action_concat_order:
569
+ - action.nav_delta
570
+ - _target_: gammanav.vln.model.wnm_3d.transform.wnm_3d_cotrain.WNM3DTransform
571
+ default_instruction: Perform the default behavior.
572
+ language_dropout_prob: 0.0
573
+ always_use_default_instruction: false
574
+ max_state_dim: 64
575
+ max_action_dim: 32
576
+ max_length: 512
577
+ state_horizon: 4
578
+ action_horizon: 8
579
+ tokenizer_path: ./checkpoints/umt5-xxl
580
+ mixture_dataset_cls: gammanav.vln.data.dataset.lerobot.LeRobotMixtureDataset.from_mixture_spec
581
+ single_dataset_cls: gammanav.vln.data.dataset.lerobot_sharded.ShardedLeRobotSingleActionChunkDatasetInteriorGS
582
+ total_training_steps: 89725000
wnm_3d_stage1/experiment_cfg/metadata.json ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "interiorgs": {
3
+ "statistics": {
4
+ "state": {
5
+ "nav_pose": {
6
+ "max": [
7
+ 0.0,
8
+ 0.0,
9
+ 0.0
10
+ ],
11
+ "min": [
12
+ 0.0,
13
+ 0.0,
14
+ 0.0
15
+ ],
16
+ "mean": [
17
+ 0.0,
18
+ 0.0,
19
+ 0.0
20
+ ],
21
+ "std": [
22
+ 0.0,
23
+ 0.0,
24
+ 0.0
25
+ ],
26
+ "q01": [
27
+ -1.0,
28
+ -1.0,
29
+ -1.0
30
+ ],
31
+ "q99": [
32
+ 1.0,
33
+ 1.0,
34
+ 1.0
35
+ ]
36
+ }
37
+ },
38
+ "action": {
39
+ "nav_delta": {
40
+ "max": [
41
+ 0.8474730849266052,
42
+ 0.8248558044433594,
43
+ 1.1043548583984375
44
+ ],
45
+ "min": [
46
+ -0.6632754802703857,
47
+ -0.7584977149963379,
48
+ -1.309535026550293
49
+ ],
50
+ "mean": [
51
+ 0.4112831652164459,
52
+ 0.02591397985816002,
53
+ 0.011686268262565136
54
+ ],
55
+ "std": [
56
+ 0.30634430050849915,
57
+ 0.25818490982055664,
58
+ 0.2255612015724182
59
+ ],
60
+ "q01": [
61
+ -0.5167848467826843,
62
+ -0.6000025272369385,
63
+ -0.7039973735809326
64
+ ],
65
+ "q99": [
66
+ 0.8178253173828125,
67
+ 0.6459061503410339,
68
+ 0.7619411945343018
69
+ ]
70
+ }
71
+ }
72
+ },
73
+ "modalities": {
74
+ "video": {
75
+ "rgb": {
76
+ "resolution": [
77
+ 480,
78
+ 360
79
+ ],
80
+ "channels": 3,
81
+ "fps": 3.3333333333333335
82
+ }
83
+ },
84
+ "state": {
85
+ "nav_pose": {
86
+ "absolute": false,
87
+ "rotation_type": null,
88
+ "shape": [
89
+ 3
90
+ ],
91
+ "continuous": true
92
+ }
93
+ },
94
+ "action": {
95
+ "nav_delta": {
96
+ "absolute": false,
97
+ "rotation_type": null,
98
+ "shape": [
99
+ 3
100
+ ],
101
+ "continuous": true
102
+ }
103
+ }
104
+ },
105
+ "embodiment_tag": "interiorgs"
106
+ }
107
+ }
wnm_3d_stage1/model-00001-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0e0ff763ed8bb6b855782e10f1577d676bc7abce2df0e46cea5d340a05963a5
3
+ size 4935816720
wnm_3d_stage1/model-00002-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3dd6a6ed2735b920badc8b58864ac567f921b5e1ab537f9c755e35e6c65ebd8c
3
+ size 4983107552
wnm_3d_stage1/model-00003-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:18bfbedaeb42968bb1f0032166eb766b58823748c25cb9d2de3aa60e47730498
3
+ size 4919126194
wnm_3d_stage1/model-00004-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:47eb0444e18d4427e060df6fa3b500b1d583c47a8591da99c25af6af8b8c3e1a
3
+ size 4997878856
wnm_3d_stage1/model-00005-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:640c7044553520c14824bd1c4ae835b881d768df0a2be28610dc2feff36f9021
3
+ size 4947510648
wnm_3d_stage1/model-00006-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cb488544ad0165d11a57790d30f95d6c050df5fbecab95eb37f496d777bf39cd
3
+ size 2329945620
wnm_3d_stage1/model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
wnm_3d_stage2/action_normalization.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema": "wnm_3d_action_normalization_v1",
3
+ "embodiment": "interiorgs",
4
+ "action_key": "action.nav_delta",
5
+ "channel_order": [
6
+ "dx_m",
7
+ "dy_m",
8
+ "dyaw_rad"
9
+ ],
10
+ "model_action_dim": 32,
11
+ "action_horizon_per_chunk": 8,
12
+ "decoded_action_dims": 3,
13
+ "normalization_mode": "q99",
14
+ "normalized_reference_range": [
15
+ -1.0,
16
+ 1.0
17
+ ],
18
+ "q01": [
19
+ -0.6557483077049255,
20
+ -0.8707880973815918,
21
+ -1.104737639427185
22
+ ],
23
+ "q99": [
24
+ 1.4973504543304443,
25
+ 0.861659824848175,
26
+ 1.2756091356277466
27
+ ],
28
+ "nav_action_scale": 4.0,
29
+ "physical_q01_after_scale": [
30
+ -0.16393707692623138,
31
+ -0.21769702434539795,
32
+ -0.27618440985679626
33
+ ],
34
+ "physical_q99_after_scale": [
35
+ 0.3743376135826111,
36
+ 0.21541495621204376,
37
+ 0.31890228390693665
38
+ ],
39
+ "decode_formula": "delta = (((action_norm[..., :3] + 1) * 0.5) * (q99 - q01) + q01) / nav_action_scale",
40
+ "encode_formula": "action_norm = 2 * ((delta * nav_action_scale) - q01) / (q99 - q01) - 1",
41
+ "decode_clamps_normalized_action": false
42
+ }
wnm_3d_stage2/config.json ADDED
@@ -0,0 +1,93 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_dim": 32,
3
+ "action_head_cfg": {
4
+ "_convert_": "object",
5
+ "_target_": "gammanav.vln.model.wnm_3d.action_head.wan_flow_matching_action_tf.WANPolicyHead",
6
+ "config": {
7
+ "_recursive_": false,
8
+ "_target_": "gammanav.vln.model.wnm_3d.action_head.wan_flow_matching_action_tf.WANPolicyHeadConfig",
9
+ "action_dim": 32,
10
+ "action_horizon": 8,
11
+ "backbone_features_projector_cfg": null,
12
+ "decouple_video_action_noise": false,
13
+ "diffusion_model_cfg": {
14
+ "_convert_": "object",
15
+ "_target_": "gammanav.vln.model.wnm_3d.modules.wan_video_dit_action_casual_chunk.CausalWanModel",
16
+ "concat_first_frame_latent": false,
17
+ "diffusion_model_pretrained_path": "./checkpoints/Wan2.2-TI2V-5B",
18
+ "dim": 3072,
19
+ "eps": 1e-06,
20
+ "ffn_dim": 14336,
21
+ "frame_seqlen": 50,
22
+ "freq_dim": 256,
23
+ "in_dim": 48,
24
+ "max_chunk_size": 4,
25
+ "model_type": "ti2v",
26
+ "num_action_per_block": 8,
27
+ "num_frame_per_block": 2,
28
+ "num_heads": 24,
29
+ "num_layers": 30,
30
+ "num_state_per_block": 1,
31
+ "out_dim": 48
32
+ },
33
+ "image_encoder_cfg": {
34
+ "_convert_": "object",
35
+ "_target_": "gammanav.vln.model.wnm_3d.modules.wan_video_image_encoder.WanImageEncoder",
36
+ "image_encoder_pretrained_path": "./checkpoints/Wan2.1-I2V-14B-480P/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"
37
+ },
38
+ "init_lora_weights": "kaiming",
39
+ "lora_alpha": 4,
40
+ "lora_rank": 4,
41
+ "lora_target_modules": "q,k,v,o,ffn.0,ffn.2",
42
+ "max_action_dim": 32,
43
+ "max_state_dim": 64,
44
+ "num_frame_per_block": 2,
45
+ "num_frames": 33,
46
+ "num_inference_timesteps": 4,
47
+ "skip_component_loading": true,
48
+ "target_video_height": 160,
49
+ "target_video_width": 320,
50
+ "text_encoder_cfg": {
51
+ "_convert_": "object",
52
+ "_target_": "gammanav.vln.model.wnm_3d.modules.wan_video_text_encoder.WanTextEncoder",
53
+ "text_encoder_pretrained_path": "./checkpoints/Wan2.2-TI2V-5B/models_t5_umt5-xxl-enc-bf16.pth"
54
+ },
55
+ "tile_size_height": 34,
56
+ "tile_size_width": 34,
57
+ "tile_stride_height": 18,
58
+ "tile_stride_width": 16,
59
+ "tiled": false,
60
+ "train_architecture": "full",
61
+ "tune_diffusion_model": true,
62
+ "tune_projector": true,
63
+ "use_vggt_geometry_adapter": true,
64
+ "vae_cfg": {
65
+ "_convert_": "object",
66
+ "_target_": "gammanav.vln.model.wnm_3d.modules.wan_video_vae.WanVideoVAE38",
67
+ "dim": 160,
68
+ "vae_pretrained_path": "./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth",
69
+ "z_dim": 48
70
+ },
71
+ "vggt_adapter_blocks": 2,
72
+ "vggt_adapter_dim": 512,
73
+ "vggt_adapter_heads": 8,
74
+ "vggt_checkpoint_path": "./checkpoints/vggt_omega_1b_512.pt",
75
+ "vggt_image_resolution": 512,
76
+ "vggt_resize_mode": "square",
77
+ "video_noise_beta_alpha": 3.0,
78
+ "video_noise_beta_beta": 1.0
79
+ }
80
+ },
81
+ "action_horizon": 8,
82
+ "architectures": [
83
+ "WNM3DModel"
84
+ ],
85
+ "backbone_cfg": {
86
+ "_target_": "gammanav.vln.model.wnm_3d.backbone.identity.IdentityBackbone"
87
+ },
88
+ "hidden_size": 0,
89
+ "model_dtype": "float32",
90
+ "model_type": "wnm_3d",
91
+ "torch_dtype": "bfloat16",
92
+ "transformers_version": "4.51.3"
93
+ }
wnm_3d_stage2/experiment_cfg/conf.yaml ADDED
@@ -0,0 +1,581 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ _target_: gammanav.vln.model.wnm_3d.model.WNM3DModel
3
+ _convert_: object
4
+ config:
5
+ _target_: gammanav.vln.model.wnm_3d.model.WNM3DConfig
6
+ _recursive_: false
7
+ model_dtype: float32
8
+ hidden_size: 0
9
+ action_horizon: 8
10
+ action_dim: 32
11
+ backbone_cfg:
12
+ _target_: gammanav.vln.model.wnm_3d.backbone.identity.IdentityBackbone
13
+ action_head_cfg:
14
+ config:
15
+ backbone_features_projector_cfg: null
16
+ _target_: gammanav.vln.model.wnm_3d.action_head.wan_flow_matching_action_tf.WANPolicyHeadConfig
17
+ _recursive_: false
18
+ tiled: false
19
+ tile_size_height: 34
20
+ tile_size_width: 34
21
+ tile_stride_height: 18
22
+ tile_stride_width: 16
23
+ lora_rank: 4
24
+ lora_alpha: 4
25
+ num_frames: 33
26
+ num_frame_per_block: 2
27
+ lora_target_modules: q,k,v,o,ffn.0,ffn.2
28
+ init_lora_weights: kaiming
29
+ train_architecture: full
30
+ use_vggt_geometry_adapter: true
31
+ vggt_checkpoint_path: ./checkpoints/vggt_omega_1b_512.pt
32
+ vggt_image_resolution: 512
33
+ vggt_resize_mode: square
34
+ vggt_adapter_dim: 512
35
+ vggt_adapter_blocks: 2
36
+ vggt_adapter_heads: 8
37
+ max_state_dim: 64
38
+ max_action_dim: 32
39
+ diffusion_model_cfg:
40
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_dit_action_casual_chunk.CausalWanModel
41
+ _convert_: object
42
+ diffusion_model_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B
43
+ model_type: ti2v
44
+ frame_seqlen: 50
45
+ dim: 3072
46
+ in_dim: 48
47
+ ffn_dim: 14336
48
+ out_dim: 48
49
+ freq_dim: 256
50
+ eps: 1.0e-06
51
+ num_heads: 24
52
+ num_layers: 30
53
+ max_chunk_size: 4
54
+ num_frame_per_block: 2
55
+ num_action_per_block: 8
56
+ num_state_per_block: 1
57
+ concat_first_frame_latent: false
58
+ text_encoder_cfg:
59
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_text_encoder.WanTextEncoder
60
+ _convert_: object
61
+ text_encoder_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B/models_t5_umt5-xxl-enc-bf16.pth
62
+ image_encoder_cfg:
63
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_image_encoder.WanImageEncoder
64
+ _convert_: object
65
+ image_encoder_pretrained_path: ./checkpoints/Wan2.1-I2V-14B-480P/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth
66
+ vae_cfg:
67
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_vae.WanVideoVAE38
68
+ _convert_: object
69
+ vae_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth
70
+ z_dim: 48
71
+ dim: 160
72
+ action_dim: 32
73
+ action_horizon: 8
74
+ num_inference_timesteps: 4
75
+ decouple_video_action_noise: false
76
+ video_noise_beta_alpha: 3.0
77
+ video_noise_beta_beta: 1.0
78
+ tune_projector: true
79
+ tune_diffusion_model: true
80
+ target_video_height: 160
81
+ target_video_width: 320
82
+ _target_: gammanav.vln.model.wnm_3d.action_head.wan_flow_matching_action_tf.WANPolicyHead
83
+ _convert_: object
84
+ train_dataset:
85
+ _target_: gammanav.vln.data.dataset.lerobot.LeRobotMixtureDataset.from_mixture_spec
86
+ _convert_: object
87
+ mixture_spec:
88
+ - dataset_path:
89
+ interiorgs:
90
+ - ./data/interiorgs_dagger_vggt_stage2
91
+ dataset_weight: 1.0
92
+ distribute_weights: true
93
+ dataset_class: gammanav.vln.data.dataset.lerobot_sharded.ShardedLeRobotSingleActionChunkDatasetInteriorGS
94
+ all_modality_configs:
95
+ interiorgs:
96
+ video:
97
+ _target_: gammanav.vln.data.dataset.ModalityConfig
98
+ delta_indices: []
99
+ eval_delta_indices:
100
+ - 0
101
+ modality_keys:
102
+ - video.rgb
103
+ state:
104
+ _target_: gammanav.vln.data.dataset.ModalityConfig
105
+ delta_indices: []
106
+ modality_keys:
107
+ - state.nav_pose
108
+ action:
109
+ _target_: gammanav.vln.data.dataset.ModalityConfig
110
+ delta_indices: []
111
+ modality_keys:
112
+ - action.nav_delta
113
+ language:
114
+ _target_: gammanav.vln.data.dataset.ModalityConfig
115
+ delta_indices:
116
+ - 0
117
+ modality_keys:
118
+ - annotation.language.language_instruction
119
+ all_transforms:
120
+ interiorgs:
121
+ _target_: gammanav.vln.data.transform.ComposedModalityTransform
122
+ transforms:
123
+ - _target_: gammanav.vln.data.transform.VideoToTensor
124
+ apply_to:
125
+ - video.rgb
126
+ - _target_: gammanav.vln.data.transform.VideoCrop
127
+ apply_to:
128
+ - video.rgb
129
+ scale: 0.95
130
+ mode: random
131
+ - _target_: gammanav.vln.data.transform.VideoResize
132
+ apply_to:
133
+ - video.rgb
134
+ height: 160
135
+ width: 320
136
+ interpolation: linear
137
+ - _target_: gammanav.vln.data.transform.VideoColorJitter
138
+ apply_to:
139
+ - video.rgb
140
+ brightness: 0.3
141
+ contrast: 0.4
142
+ saturation: 0.5
143
+ hue: 0.08
144
+ - _target_: gammanav.vln.data.transform.VideoToNumpy
145
+ apply_to:
146
+ - video.rgb
147
+ - _target_: gammanav.vln.data.transform.StateActionToTensor
148
+ apply_to:
149
+ - state.nav_pose
150
+ - _target_: gammanav.vln.data.transform.StateActionTransform
151
+ apply_to:
152
+ - state.nav_pose
153
+ normalization_modes:
154
+ state.nav_pose: q99
155
+ - _target_: gammanav.vln.data.transform.StateActionToTensor
156
+ apply_to:
157
+ - action.nav_delta
158
+ - _target_: gammanav.vln.data.transform.StateActionTransform
159
+ apply_to:
160
+ - action.nav_delta
161
+ normalization_modes:
162
+ action.nav_delta: q99
163
+ - _target_: gammanav.vln.data.transform.ConcatTransform
164
+ video_concat_order:
165
+ - video.rgb
166
+ state_concat_order:
167
+ - state.nav_pose
168
+ action_concat_order:
169
+ - action.nav_delta
170
+ - _target_: gammanav.vln.model.wnm_3d.transform.wnm_3d_cotrain.WNM3DTransform
171
+ default_instruction: Perform the default behavior.
172
+ language_dropout_prob: 0.0
173
+ always_use_default_instruction: false
174
+ max_state_dim: 64
175
+ max_action_dim: 32
176
+ max_length: 512
177
+ state_horizon: 4
178
+ action_horizon: 8
179
+ tokenizer_path: ./checkpoints/umt5-xxl
180
+ dataset_kwargs:
181
+ max_chunk_size: 4
182
+ num_frames: 33
183
+ action_horizon: 8
184
+ state_horizon: 4
185
+ num_frame_per_block: 2
186
+ num_action_per_block: 8
187
+ num_state_per_block: 1
188
+ vae_temporal_stride: 4
189
+ video_frame_stride: 1
190
+ action_frame_stride: 1
191
+ history_sampling: uniform
192
+ nav_action_scale: 4.0
193
+ nav_smooth_window: 7
194
+ nav_smooth_passes: 2
195
+ compute_statistics: true
196
+ max_stats_episodes: 512
197
+ stats_start_stride: 16
198
+ mixture_kwargs:
199
+ training: true
200
+ balance_dataset_weights: false
201
+ balance_trajectory_weights: true
202
+ seed: 42
203
+ allow_padding_at_end: true
204
+ trainer:
205
+ _target_: gammanav.vln.experiment.VLNTrainer
206
+ _partial_: true
207
+ _recursive_: false
208
+ callbacks: null
209
+ model: ???
210
+ train_dataset: ???
211
+ compute_dtype: ???
212
+ wandb_project: wnm_3d
213
+ output_dir: ./checkpoints/wnm_3d_stage2
214
+ load_from_yaml: null
215
+ remove_unused_columns: false
216
+ bf16: true
217
+ tf32: true
218
+ global_batch_size: 192
219
+ raise_error_if_global_batch_size_not_set: false
220
+ per_device_train_batch_size: 12
221
+ per_device_eval_batch_size: 64
222
+ gradient_accumulation_steps: 1
223
+ dataloader_num_workers: 4
224
+ dataloader_pin_memory: false
225
+ dataloader_persistent_workers: true
226
+ optim: adamw_torch
227
+ learning_rate: 0.0001
228
+ adam_beta1: 0.95
229
+ adam_beta2: 0.999
230
+ adam_epsilon: 1.0e-08
231
+ weight_decay: 1.0e-05
232
+ lr_scheduler_type: cosine
233
+ warmup_ratio: 0.05
234
+ logging_steps: 10
235
+ num_train_epochs: 1000
236
+ max_steps: 35000
237
+ save_strategy: steps
238
+ save_steps: 1000
239
+ eval_strategy: 'no'
240
+ save_total_limit: 10
241
+ report_to: tensorboard
242
+ seed: 42
243
+ do_eval: false
244
+ gradient_checkpointing: false
245
+ ddp_find_unused_parameters: false
246
+ ddp_bucket_cap_mb: 100
247
+ eval_bf16: true
248
+ torch_compile_mode: null
249
+ pretrained_model_path: ./checkpoints/wnm_3d_stage1
250
+ save_lora_only: false
251
+ training_args:
252
+ _target_: transformers.TrainingArguments
253
+ output_dir: ./checkpoints/wnm_3d_stage2
254
+ run_name: wnm_3d_stage2
255
+ remove_unused_columns: false
256
+ deepspeed: gammanav/vln/configs/deepspeed/zero2.json
257
+ gradient_checkpointing: false
258
+ bf16: true
259
+ tf32: true
260
+ per_device_train_batch_size: 12
261
+ per_device_eval_batch_size: 64
262
+ gradient_accumulation_steps: 1
263
+ dataloader_num_workers: 4
264
+ dataloader_pin_memory: false
265
+ dataloader_persistent_workers: true
266
+ optim: adamw_torch
267
+ adam_beta1: 0.95
268
+ adam_beta2: 0.999
269
+ adam_epsilon: 1.0e-08
270
+ learning_rate: 1.0e-05
271
+ weight_decay: 1.0e-05
272
+ warmup_ratio: 0.05
273
+ lr_scheduler_type: cosine
274
+ logging_steps: 10
275
+ num_train_epochs: 1000
276
+ max_steps: 35000
277
+ save_strategy: steps
278
+ save_steps: 1000
279
+ save_total_limit: 10
280
+ report_to: tensorboard
281
+ seed: 42
282
+ do_eval: false
283
+ ddp_find_unused_parameters: false
284
+ ddp_bucket_cap_mb: 100
285
+ torch_compile_mode: null
286
+ logging_dir: ./checkpoints/wnm_3d_stage2/tensorboard
287
+ backbone_hidden_size: 0
288
+ backbone_cfg:
289
+ _target_: gammanav.vln.model.wnm_3d.backbone.identity.IdentityBackbone
290
+ action_head_cfg:
291
+ config:
292
+ backbone_features_projector_cfg: null
293
+ _target_: gammanav.vln.model.wnm_3d.action_head.wan_flow_matching_action_tf.WANPolicyHeadConfig
294
+ _recursive_: false
295
+ tiled: false
296
+ tile_size_height: 34
297
+ tile_size_width: 34
298
+ tile_stride_height: 18
299
+ tile_stride_width: 16
300
+ lora_rank: 4
301
+ lora_alpha: 4
302
+ num_frames: 33
303
+ num_frame_per_block: 2
304
+ lora_target_modules: q,k,v,o,ffn.0,ffn.2
305
+ init_lora_weights: kaiming
306
+ train_architecture: full
307
+ use_vggt_geometry_adapter: true
308
+ vggt_checkpoint_path: ./checkpoints/vggt_omega_1b_512.pt
309
+ vggt_image_resolution: 512
310
+ vggt_resize_mode: square
311
+ vggt_adapter_dim: 512
312
+ vggt_adapter_blocks: 2
313
+ vggt_adapter_heads: 8
314
+ max_state_dim: 64
315
+ max_action_dim: 32
316
+ diffusion_model_cfg:
317
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_dit_action_casual_chunk.CausalWanModel
318
+ _convert_: object
319
+ diffusion_model_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B
320
+ model_type: ti2v
321
+ frame_seqlen: 50
322
+ dim: 3072
323
+ in_dim: 48
324
+ ffn_dim: 14336
325
+ out_dim: 48
326
+ freq_dim: 256
327
+ eps: 1.0e-06
328
+ num_heads: 24
329
+ num_layers: 30
330
+ max_chunk_size: 4
331
+ num_frame_per_block: 2
332
+ num_action_per_block: 8
333
+ num_state_per_block: 1
334
+ concat_first_frame_latent: false
335
+ text_encoder_cfg:
336
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_text_encoder.WanTextEncoder
337
+ _convert_: object
338
+ text_encoder_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B/models_t5_umt5-xxl-enc-bf16.pth
339
+ image_encoder_cfg:
340
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_image_encoder.WanImageEncoder
341
+ _convert_: object
342
+ image_encoder_pretrained_path: ./checkpoints/Wan2.1-I2V-14B-480P/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth
343
+ vae_cfg:
344
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_vae.WanVideoVAE38
345
+ _convert_: object
346
+ vae_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth
347
+ z_dim: 48
348
+ dim: 160
349
+ action_dim: 32
350
+ action_horizon: 8
351
+ num_inference_timesteps: 4
352
+ decouple_video_action_noise: false
353
+ video_noise_beta_alpha: 3.0
354
+ video_noise_beta_beta: 1.0
355
+ tune_projector: true
356
+ tune_diffusion_model: true
357
+ target_video_height: 160
358
+ target_video_width: 320
359
+ _target_: gammanav.vln.model.wnm_3d.action_head.wan_flow_matching_action_tf.WANPolicyHead
360
+ _convert_: object
361
+ dit_version: ./checkpoints/Wan2.2-TI2V-5B
362
+ text_encoder_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B/models_t5_umt5-xxl-enc-bf16.pth
363
+ image_encoder_pretrained_path: ./checkpoints/Wan2.1-I2V-14B-480P/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth
364
+ vae_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth
365
+ train_architecture: full
366
+ num_frame_per_block: 2
367
+ num_action_per_block: 8
368
+ num_state_per_block: 1
369
+ frame_seqlen: 50
370
+ use_vggt_geometry_adapter: true
371
+ vggt_omega_checkpoint_path: ./checkpoints/vggt_omega_1b_512.pt
372
+ vggt_image_resolution: 512
373
+ vggt_resize_mode: square
374
+ vggt_adapter_dim: 512
375
+ vggt_adapter_blocks: 2
376
+ vggt_adapter_heads: 8
377
+ max_length: 512
378
+ tokenizer_path: ./checkpoints/umt5-xxl
379
+ data_collator:
380
+ _target_: gammanav.vln.model.wnm_3d.transform.wnm_3d_cotrain.DefaultDataCollator
381
+ tokenizer_path: ./checkpoints/umt5-xxl
382
+ max_length: 512
383
+ num_visual_tokens_per_frame: 16
384
+ max_state_dim: 64
385
+ max_action_dim: 32
386
+ language_dropout_prob: 0.0
387
+ model_specific_transform:
388
+ _target_: gammanav.vln.model.wnm_3d.transform.wnm_3d_cotrain.WNM3DTransform
389
+ default_instruction: Perform the default behavior.
390
+ language_dropout_prob: 0.0
391
+ always_use_default_instruction: false
392
+ max_state_dim: 64
393
+ max_action_dim: 32
394
+ max_length: 512
395
+ state_horizon: 4
396
+ action_horizon: 8
397
+ tokenizer_path: ./checkpoints/umt5-xxl
398
+ image_resolution_width: 320
399
+ image_resolution_height: 160
400
+ num_frames: 33
401
+ state_horizon: 4
402
+ action_horizon: 8
403
+ vae_temporal_stride: 4
404
+ max_chunk_size: 4
405
+ video_frame_stride: 1
406
+ action_frame_stride: 1
407
+ history_sampling: uniform
408
+ interiorgs_data_root: ./data/interiorgs_dagger_vggt_stage2
409
+ modality_config_interiorgs:
410
+ video:
411
+ _target_: gammanav.vln.data.dataset.ModalityConfig
412
+ delta_indices: []
413
+ eval_delta_indices:
414
+ - 0
415
+ modality_keys:
416
+ - video.rgb
417
+ state:
418
+ _target_: gammanav.vln.data.dataset.ModalityConfig
419
+ delta_indices: []
420
+ modality_keys:
421
+ - state.nav_pose
422
+ action:
423
+ _target_: gammanav.vln.data.dataset.ModalityConfig
424
+ delta_indices: []
425
+ modality_keys:
426
+ - action.nav_delta
427
+ language:
428
+ _target_: gammanav.vln.data.dataset.ModalityConfig
429
+ delta_indices:
430
+ - 0
431
+ modality_keys:
432
+ - annotation.language.language_instruction
433
+ transform_interiorgs:
434
+ _target_: gammanav.vln.data.transform.ComposedModalityTransform
435
+ transforms:
436
+ - _target_: gammanav.vln.data.transform.VideoToTensor
437
+ apply_to:
438
+ - video.rgb
439
+ - _target_: gammanav.vln.data.transform.VideoCrop
440
+ apply_to:
441
+ - video.rgb
442
+ scale: 0.95
443
+ mode: random
444
+ - _target_: gammanav.vln.data.transform.VideoResize
445
+ apply_to:
446
+ - video.rgb
447
+ height: 160
448
+ width: 320
449
+ interpolation: linear
450
+ - _target_: gammanav.vln.data.transform.VideoColorJitter
451
+ apply_to:
452
+ - video.rgb
453
+ brightness: 0.3
454
+ contrast: 0.4
455
+ saturation: 0.5
456
+ hue: 0.08
457
+ - _target_: gammanav.vln.data.transform.VideoToNumpy
458
+ apply_to:
459
+ - video.rgb
460
+ - _target_: gammanav.vln.data.transform.StateActionToTensor
461
+ apply_to:
462
+ - state.nav_pose
463
+ - _target_: gammanav.vln.data.transform.StateActionTransform
464
+ apply_to:
465
+ - state.nav_pose
466
+ normalization_modes:
467
+ state.nav_pose: q99
468
+ - _target_: gammanav.vln.data.transform.StateActionToTensor
469
+ apply_to:
470
+ - action.nav_delta
471
+ - _target_: gammanav.vln.data.transform.StateActionTransform
472
+ apply_to:
473
+ - action.nav_delta
474
+ normalization_modes:
475
+ action.nav_delta: q99
476
+ - _target_: gammanav.vln.data.transform.ConcatTransform
477
+ video_concat_order:
478
+ - video.rgb
479
+ state_concat_order:
480
+ - state.nav_pose
481
+ action_concat_order:
482
+ - action.nav_delta
483
+ - _target_: gammanav.vln.model.wnm_3d.transform.wnm_3d_cotrain.WNM3DTransform
484
+ default_instruction: Perform the default behavior.
485
+ language_dropout_prob: 0.0
486
+ always_use_default_instruction: false
487
+ max_state_dim: 64
488
+ max_action_dim: 32
489
+ max_length: 512
490
+ state_horizon: 4
491
+ action_horizon: 8
492
+ tokenizer_path: ./checkpoints/umt5-xxl
493
+ modality_configs:
494
+ interiorgs:
495
+ video:
496
+ _target_: gammanav.vln.data.dataset.ModalityConfig
497
+ delta_indices: []
498
+ eval_delta_indices:
499
+ - 0
500
+ modality_keys:
501
+ - video.rgb
502
+ state:
503
+ _target_: gammanav.vln.data.dataset.ModalityConfig
504
+ delta_indices: []
505
+ modality_keys:
506
+ - state.nav_pose
507
+ action:
508
+ _target_: gammanav.vln.data.dataset.ModalityConfig
509
+ delta_indices: []
510
+ modality_keys:
511
+ - action.nav_delta
512
+ language:
513
+ _target_: gammanav.vln.data.dataset.ModalityConfig
514
+ delta_indices:
515
+ - 0
516
+ modality_keys:
517
+ - annotation.language.language_instruction
518
+ transforms:
519
+ interiorgs:
520
+ _target_: gammanav.vln.data.transform.ComposedModalityTransform
521
+ transforms:
522
+ - _target_: gammanav.vln.data.transform.VideoToTensor
523
+ apply_to:
524
+ - video.rgb
525
+ - _target_: gammanav.vln.data.transform.VideoCrop
526
+ apply_to:
527
+ - video.rgb
528
+ scale: 0.95
529
+ mode: random
530
+ - _target_: gammanav.vln.data.transform.VideoResize
531
+ apply_to:
532
+ - video.rgb
533
+ height: 160
534
+ width: 320
535
+ interpolation: linear
536
+ - _target_: gammanav.vln.data.transform.VideoColorJitter
537
+ apply_to:
538
+ - video.rgb
539
+ brightness: 0.3
540
+ contrast: 0.4
541
+ saturation: 0.5
542
+ hue: 0.08
543
+ - _target_: gammanav.vln.data.transform.VideoToNumpy
544
+ apply_to:
545
+ - video.rgb
546
+ - _target_: gammanav.vln.data.transform.StateActionToTensor
547
+ apply_to:
548
+ - state.nav_pose
549
+ - _target_: gammanav.vln.data.transform.StateActionTransform
550
+ apply_to:
551
+ - state.nav_pose
552
+ normalization_modes:
553
+ state.nav_pose: q99
554
+ - _target_: gammanav.vln.data.transform.StateActionToTensor
555
+ apply_to:
556
+ - action.nav_delta
557
+ - _target_: gammanav.vln.data.transform.StateActionTransform
558
+ apply_to:
559
+ - action.nav_delta
560
+ normalization_modes:
561
+ action.nav_delta: q99
562
+ - _target_: gammanav.vln.data.transform.ConcatTransform
563
+ video_concat_order:
564
+ - video.rgb
565
+ state_concat_order:
566
+ - state.nav_pose
567
+ action_concat_order:
568
+ - action.nav_delta
569
+ - _target_: gammanav.vln.model.wnm_3d.transform.wnm_3d_cotrain.WNM3DTransform
570
+ default_instruction: Perform the default behavior.
571
+ language_dropout_prob: 0.0
572
+ always_use_default_instruction: false
573
+ max_state_dim: 64
574
+ max_action_dim: 32
575
+ max_length: 512
576
+ state_horizon: 4
577
+ action_horizon: 8
578
+ tokenizer_path: ./checkpoints/umt5-xxl
579
+ mixture_dataset_cls: gammanav.vln.data.dataset.lerobot.LeRobotMixtureDataset.from_mixture_spec
580
+ single_dataset_cls: gammanav.vln.data.dataset.lerobot_sharded.ShardedLeRobotSingleActionChunkDatasetInteriorGS
581
+ total_training_steps: 3298000
wnm_3d_stage2/experiment_cfg/metadata.json ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "interiorgs": {
3
+ "statistics": {
4
+ "state": {
5
+ "nav_pose": {
6
+ "max": [
7
+ 0.0,
8
+ 0.0,
9
+ 0.0
10
+ ],
11
+ "min": [
12
+ 0.0,
13
+ 0.0,
14
+ 0.0
15
+ ],
16
+ "mean": [
17
+ 0.0,
18
+ 0.0,
19
+ 0.0
20
+ ],
21
+ "std": [
22
+ 0.0,
23
+ 0.0,
24
+ 0.0
25
+ ],
26
+ "q01": [
27
+ -1.0,
28
+ -1.0,
29
+ -1.0
30
+ ],
31
+ "q99": [
32
+ 1.0,
33
+ 1.0,
34
+ 1.0
35
+ ]
36
+ }
37
+ },
38
+ "action": {
39
+ "nav_delta": {
40
+ "max": [
41
+ 4.734772682189941,
42
+ 2.703505516052246,
43
+ 12.561675071716309
44
+ ],
45
+ "min": [
46
+ -4.257997035980225,
47
+ -3.7617416381835938,
48
+ -12.553963661193848
49
+ ],
50
+ "mean": [
51
+ 0.24795560538768768,
52
+ 0.006410803645849228,
53
+ 0.017522765323519707
54
+ ],
55
+ "std": [
56
+ 0.3869488537311554,
57
+ 0.3040892779827118,
58
+ 0.7763124704360962
59
+ ],
60
+ "q01": [
61
+ -0.6557483077049255,
62
+ -0.8707880973815918,
63
+ -1.104737639427185
64
+ ],
65
+ "q99": [
66
+ 1.4973504543304443,
67
+ 0.861659824848175,
68
+ 1.2756091356277466
69
+ ]
70
+ }
71
+ }
72
+ },
73
+ "modalities": {
74
+ "video": {
75
+ "rgb": {
76
+ "resolution": [
77
+ 480,
78
+ 360
79
+ ],
80
+ "channels": 3,
81
+ "fps": 3.3333333333333335
82
+ }
83
+ },
84
+ "state": {
85
+ "nav_pose": {
86
+ "absolute": false,
87
+ "rotation_type": null,
88
+ "shape": [
89
+ 3
90
+ ],
91
+ "continuous": true
92
+ }
93
+ },
94
+ "action": {
95
+ "nav_delta": {
96
+ "absolute": false,
97
+ "rotation_type": null,
98
+ "shape": [
99
+ 3
100
+ ],
101
+ "continuous": true
102
+ }
103
+ }
104
+ },
105
+ "embodiment_tag": "interiorgs"
106
+ }
107
+ }
wnm_3d_stage2/model-00001-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0e0ff763ed8bb6b855782e10f1577d676bc7abce2df0e46cea5d340a05963a5
3
+ size 4935816720
wnm_3d_stage2/model-00002-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3dd6a6ed2735b920badc8b58864ac567f921b5e1ab537f9c755e35e6c65ebd8c
3
+ size 4983107552
wnm_3d_stage2/model-00003-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fb0a94dec60bd939997f292482c15eafe77f1e200843a170decc75930d1ade8a
3
+ size 4919126194
wnm_3d_stage2/model-00004-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c15333979edef364d031550da4663cd7106503d1fbe6a24e24ef394493444ae9
3
+ size 4997878856
wnm_3d_stage2/model-00005-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:21cc0a95a40a75d4d66fe32d3da89ddc95c1275eaaad662aa5a4fe6f5a0cb788
3
+ size 4947510648
wnm_3d_stage2/model-00006-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:34876118797cb7fd9b0efcb98e709879f442a04f81e7a3d1dbdce89185dd57cf
3
+ size 2329945620
wnm_3d_stage2/model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
wnm_3d_stage3/action_normalization.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema": "wnm_3d_action_normalization_v1",
3
+ "embodiment": "interiorgs",
4
+ "action_key": "action.nav_delta",
5
+ "channel_order": [
6
+ "dx_m",
7
+ "dy_m",
8
+ "dyaw_rad"
9
+ ],
10
+ "model_action_dim": 32,
11
+ "action_horizon_per_chunk": 8,
12
+ "decoded_action_dims": 3,
13
+ "normalization_mode": "q99",
14
+ "normalized_reference_range": [
15
+ -1.0,
16
+ 1.0
17
+ ],
18
+ "q01": [
19
+ -0.6557483077049255,
20
+ -0.8707880973815918,
21
+ -1.104737639427185
22
+ ],
23
+ "q99": [
24
+ 1.4973504543304443,
25
+ 0.861659824848175,
26
+ 1.2756091356277466
27
+ ],
28
+ "nav_action_scale": 4.0,
29
+ "physical_q01_after_scale": [
30
+ -0.16393707692623138,
31
+ -0.21769702434539795,
32
+ -0.27618440985679626
33
+ ],
34
+ "physical_q99_after_scale": [
35
+ 0.3743376135826111,
36
+ 0.21541495621204376,
37
+ 0.31890228390693665
38
+ ],
39
+ "decode_formula": "delta = (((action_norm[..., :3] + 1) * 0.5) * (q99 - q01) + q01) / nav_action_scale",
40
+ "encode_formula": "action_norm = 2 * ((delta * nav_action_scale) - q01) / (q99 - q01) - 1",
41
+ "decode_clamps_normalized_action": false
42
+ }
wnm_3d_stage3/config.json ADDED
@@ -0,0 +1,93 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_dim": 32,
3
+ "action_head_cfg": {
4
+ "_convert_": "object",
5
+ "_target_": "gammanav.vln.model.wnm_3d.action_head.wan_flow_matching_action_tf.WANPolicyHead",
6
+ "config": {
7
+ "_recursive_": false,
8
+ "_target_": "gammanav.vln.model.wnm_3d.action_head.wan_flow_matching_action_tf.WANPolicyHeadConfig",
9
+ "action_dim": 32,
10
+ "action_horizon": 8,
11
+ "backbone_features_projector_cfg": null,
12
+ "decouple_video_action_noise": false,
13
+ "diffusion_model_cfg": {
14
+ "_convert_": "object",
15
+ "_target_": "gammanav.vln.model.wnm_3d.modules.wan_video_dit_action_casual_chunk.CausalWanModel",
16
+ "concat_first_frame_latent": false,
17
+ "diffusion_model_pretrained_path": "./checkpoints/Wan2.2-TI2V-5B",
18
+ "dim": 3072,
19
+ "eps": 1e-06,
20
+ "ffn_dim": 14336,
21
+ "frame_seqlen": 50,
22
+ "freq_dim": 256,
23
+ "in_dim": 48,
24
+ "max_chunk_size": 4,
25
+ "model_type": "ti2v",
26
+ "num_action_per_block": 8,
27
+ "num_frame_per_block": 2,
28
+ "num_heads": 24,
29
+ "num_layers": 30,
30
+ "num_state_per_block": 1,
31
+ "out_dim": 48
32
+ },
33
+ "image_encoder_cfg": {
34
+ "_convert_": "object",
35
+ "_target_": "gammanav.vln.model.wnm_3d.modules.wan_video_image_encoder.WanImageEncoder",
36
+ "image_encoder_pretrained_path": "./checkpoints/Wan2.1-I2V-14B-480P/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"
37
+ },
38
+ "init_lora_weights": "kaiming",
39
+ "lora_alpha": 4,
40
+ "lora_rank": 4,
41
+ "lora_target_modules": "q,k,v,o,ffn.0,ffn.2",
42
+ "max_action_dim": 32,
43
+ "max_state_dim": 64,
44
+ "num_frame_per_block": 2,
45
+ "num_frames": 33,
46
+ "num_inference_timesteps": 4,
47
+ "skip_component_loading": true,
48
+ "target_video_height": 160,
49
+ "target_video_width": 320,
50
+ "text_encoder_cfg": {
51
+ "_convert_": "object",
52
+ "_target_": "gammanav.vln.model.wnm_3d.modules.wan_video_text_encoder.WanTextEncoder",
53
+ "text_encoder_pretrained_path": "./checkpoints/Wan2.2-TI2V-5B/models_t5_umt5-xxl-enc-bf16.pth"
54
+ },
55
+ "tile_size_height": 34,
56
+ "tile_size_width": 34,
57
+ "tile_stride_height": 18,
58
+ "tile_stride_width": 16,
59
+ "tiled": false,
60
+ "train_architecture": "full",
61
+ "tune_diffusion_model": true,
62
+ "tune_projector": true,
63
+ "use_vggt_geometry_adapter": true,
64
+ "vae_cfg": {
65
+ "_convert_": "object",
66
+ "_target_": "gammanav.vln.model.wnm_3d.modules.wan_video_vae.WanVideoVAE38",
67
+ "dim": 160,
68
+ "vae_pretrained_path": "./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth",
69
+ "z_dim": 48
70
+ },
71
+ "vggt_adapter_blocks": 2,
72
+ "vggt_adapter_dim": 512,
73
+ "vggt_adapter_heads": 8,
74
+ "vggt_checkpoint_path": "./checkpoints/vggt_omega_1b_512.pt",
75
+ "vggt_image_resolution": 512,
76
+ "vggt_resize_mode": "square",
77
+ "video_noise_beta_alpha": 3.0,
78
+ "video_noise_beta_beta": 1.0
79
+ }
80
+ },
81
+ "action_horizon": 8,
82
+ "architectures": [
83
+ "WNM3DModel"
84
+ ],
85
+ "backbone_cfg": {
86
+ "_target_": "gammanav.vln.model.wnm_3d.backbone.identity.IdentityBackbone"
87
+ },
88
+ "hidden_size": 0,
89
+ "model_dtype": "float32",
90
+ "model_type": "wnm_3d",
91
+ "torch_dtype": "bfloat16",
92
+ "transformers_version": "4.51.3"
93
+ }
wnm_3d_stage3/experiment_cfg/conf.yaml ADDED
@@ -0,0 +1,581 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ _target_: gammanav.vln.model.wnm_3d.model.WNM3DModel
3
+ _convert_: object
4
+ config:
5
+ _target_: gammanav.vln.model.wnm_3d.model.WNM3DConfig
6
+ _recursive_: false
7
+ model_dtype: float32
8
+ hidden_size: 0
9
+ action_horizon: 8
10
+ action_dim: 32
11
+ backbone_cfg:
12
+ _target_: gammanav.vln.model.wnm_3d.backbone.identity.IdentityBackbone
13
+ action_head_cfg:
14
+ config:
15
+ backbone_features_projector_cfg: null
16
+ _target_: gammanav.vln.model.wnm_3d.action_head.wan_flow_matching_action_tf.WANPolicyHeadConfig
17
+ _recursive_: false
18
+ tiled: false
19
+ tile_size_height: 34
20
+ tile_size_width: 34
21
+ tile_stride_height: 18
22
+ tile_stride_width: 16
23
+ lora_rank: 4
24
+ lora_alpha: 4
25
+ num_frames: 33
26
+ num_frame_per_block: 2
27
+ lora_target_modules: q,k,v,o,ffn.0,ffn.2
28
+ init_lora_weights: kaiming
29
+ train_architecture: full
30
+ use_vggt_geometry_adapter: true
31
+ vggt_checkpoint_path: ./checkpoints/vggt_omega_1b_512.pt
32
+ vggt_image_resolution: 512
33
+ vggt_resize_mode: square
34
+ vggt_adapter_dim: 512
35
+ vggt_adapter_blocks: 2
36
+ vggt_adapter_heads: 8
37
+ max_state_dim: 64
38
+ max_action_dim: 32
39
+ diffusion_model_cfg:
40
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_dit_action_casual_chunk.CausalWanModel
41
+ _convert_: object
42
+ diffusion_model_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B
43
+ model_type: ti2v
44
+ frame_seqlen: 50
45
+ dim: 3072
46
+ in_dim: 48
47
+ ffn_dim: 14336
48
+ out_dim: 48
49
+ freq_dim: 256
50
+ eps: 1.0e-06
51
+ num_heads: 24
52
+ num_layers: 30
53
+ max_chunk_size: 4
54
+ num_frame_per_block: 2
55
+ num_action_per_block: 8
56
+ num_state_per_block: 1
57
+ concat_first_frame_latent: false
58
+ text_encoder_cfg:
59
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_text_encoder.WanTextEncoder
60
+ _convert_: object
61
+ text_encoder_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B/models_t5_umt5-xxl-enc-bf16.pth
62
+ image_encoder_cfg:
63
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_image_encoder.WanImageEncoder
64
+ _convert_: object
65
+ image_encoder_pretrained_path: ./checkpoints/Wan2.1-I2V-14B-480P/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth
66
+ vae_cfg:
67
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_vae.WanVideoVAE38
68
+ _convert_: object
69
+ vae_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth
70
+ z_dim: 48
71
+ dim: 160
72
+ action_dim: 32
73
+ action_horizon: 8
74
+ num_inference_timesteps: 4
75
+ decouple_video_action_noise: false
76
+ video_noise_beta_alpha: 3.0
77
+ video_noise_beta_beta: 1.0
78
+ tune_projector: true
79
+ tune_diffusion_model: true
80
+ target_video_height: 160
81
+ target_video_width: 320
82
+ _target_: gammanav.vln.model.wnm_3d.action_head.wan_flow_matching_action_tf.WANPolicyHead
83
+ _convert_: object
84
+ train_dataset:
85
+ _target_: gammanav.vln.data.dataset.lerobot.LeRobotMixtureDataset.from_mixture_spec
86
+ _convert_: object
87
+ mixture_spec:
88
+ - dataset_path:
89
+ interiorgs:
90
+ - ./data/interiorgs_lerobot_seen
91
+ dataset_weight: 1.0
92
+ distribute_weights: true
93
+ dataset_class: gammanav.vln.data.dataset.lerobot_sharded.ShardedLeRobotSingleActionChunkDatasetInteriorGS
94
+ all_modality_configs:
95
+ interiorgs:
96
+ video:
97
+ _target_: gammanav.vln.data.dataset.ModalityConfig
98
+ delta_indices: []
99
+ eval_delta_indices:
100
+ - 0
101
+ modality_keys:
102
+ - video.rgb
103
+ state:
104
+ _target_: gammanav.vln.data.dataset.ModalityConfig
105
+ delta_indices: []
106
+ modality_keys:
107
+ - state.nav_pose
108
+ action:
109
+ _target_: gammanav.vln.data.dataset.ModalityConfig
110
+ delta_indices: []
111
+ modality_keys:
112
+ - action.nav_delta
113
+ language:
114
+ _target_: gammanav.vln.data.dataset.ModalityConfig
115
+ delta_indices:
116
+ - 0
117
+ modality_keys:
118
+ - annotation.language.language_instruction
119
+ all_transforms:
120
+ interiorgs:
121
+ _target_: gammanav.vln.data.transform.ComposedModalityTransform
122
+ transforms:
123
+ - _target_: gammanav.vln.data.transform.VideoToTensor
124
+ apply_to:
125
+ - video.rgb
126
+ - _target_: gammanav.vln.data.transform.VideoCrop
127
+ apply_to:
128
+ - video.rgb
129
+ scale: 0.95
130
+ mode: random
131
+ - _target_: gammanav.vln.data.transform.VideoResize
132
+ apply_to:
133
+ - video.rgb
134
+ height: 160
135
+ width: 320
136
+ interpolation: linear
137
+ - _target_: gammanav.vln.data.transform.VideoColorJitter
138
+ apply_to:
139
+ - video.rgb
140
+ brightness: 0.3
141
+ contrast: 0.4
142
+ saturation: 0.5
143
+ hue: 0.08
144
+ - _target_: gammanav.vln.data.transform.VideoToNumpy
145
+ apply_to:
146
+ - video.rgb
147
+ - _target_: gammanav.vln.data.transform.StateActionToTensor
148
+ apply_to:
149
+ - state.nav_pose
150
+ - _target_: gammanav.vln.data.transform.StateActionTransform
151
+ apply_to:
152
+ - state.nav_pose
153
+ normalization_modes:
154
+ state.nav_pose: q99
155
+ - _target_: gammanav.vln.data.transform.StateActionToTensor
156
+ apply_to:
157
+ - action.nav_delta
158
+ - _target_: gammanav.vln.data.transform.StateActionTransform
159
+ apply_to:
160
+ - action.nav_delta
161
+ normalization_modes:
162
+ action.nav_delta: q99
163
+ - _target_: gammanav.vln.data.transform.ConcatTransform
164
+ video_concat_order:
165
+ - video.rgb
166
+ state_concat_order:
167
+ - state.nav_pose
168
+ action_concat_order:
169
+ - action.nav_delta
170
+ - _target_: gammanav.vln.model.wnm_3d.transform.wnm_3d_cotrain.WNM3DTransform
171
+ default_instruction: Perform the default behavior.
172
+ language_dropout_prob: 0.0
173
+ always_use_default_instruction: false
174
+ max_state_dim: 64
175
+ max_action_dim: 32
176
+ max_length: 512
177
+ state_horizon: 4
178
+ action_horizon: 8
179
+ tokenizer_path: ./checkpoints/umt5-xxl
180
+ dataset_kwargs:
181
+ max_chunk_size: 4
182
+ num_frames: 33
183
+ action_horizon: 8
184
+ state_horizon: 4
185
+ num_frame_per_block: 2
186
+ num_action_per_block: 8
187
+ num_state_per_block: 1
188
+ vae_temporal_stride: 4
189
+ video_frame_stride: 1
190
+ action_frame_stride: 1
191
+ history_sampling: uniform
192
+ nav_action_scale: 4.0
193
+ nav_smooth_window: 7
194
+ nav_smooth_passes: 2
195
+ compute_statistics: true
196
+ max_stats_episodes: 512
197
+ stats_start_stride: 16
198
+ mixture_kwargs:
199
+ training: true
200
+ balance_dataset_weights: false
201
+ balance_trajectory_weights: true
202
+ seed: 42
203
+ allow_padding_at_end: true
204
+ trainer:
205
+ _target_: gammanav.vln.experiment.VLNTrainer
206
+ _partial_: true
207
+ _recursive_: false
208
+ callbacks: null
209
+ model: ???
210
+ train_dataset: ???
211
+ compute_dtype: ???
212
+ wandb_project: wnm_3d
213
+ output_dir: ./checkpoints/wnm_3d_stage3
214
+ load_from_yaml: null
215
+ remove_unused_columns: false
216
+ bf16: true
217
+ tf32: true
218
+ global_batch_size: 8
219
+ raise_error_if_global_batch_size_not_set: false
220
+ per_device_train_batch_size: 1
221
+ per_device_eval_batch_size: 64
222
+ gradient_accumulation_steps: 1
223
+ dataloader_num_workers: 0
224
+ dataloader_pin_memory: false
225
+ dataloader_persistent_workers: false
226
+ optim: adamw_torch
227
+ learning_rate: 0.0001
228
+ adam_beta1: 0.95
229
+ adam_beta2: 0.999
230
+ adam_epsilon: 1.0e-08
231
+ weight_decay: 1.0e-05
232
+ lr_scheduler_type: cosine
233
+ warmup_ratio: 0.05
234
+ logging_steps: 1
235
+ num_train_epochs: 1000
236
+ max_steps: 10
237
+ save_strategy: steps
238
+ save_steps: 10
239
+ eval_strategy: 'no'
240
+ save_total_limit: 5
241
+ report_to: none
242
+ seed: 42
243
+ do_eval: false
244
+ gradient_checkpointing: false
245
+ ddp_find_unused_parameters: false
246
+ ddp_bucket_cap_mb: 100
247
+ eval_bf16: true
248
+ torch_compile_mode: null
249
+ pretrained_model_path: null
250
+ save_lora_only: false
251
+ training_args:
252
+ _target_: transformers.TrainingArguments
253
+ output_dir: ./checkpoints/wnm_3d_stage3
254
+ run_name: wnm_3d_10step_reference
255
+ remove_unused_columns: false
256
+ deepspeed: gammanav/vln/configs/deepspeed/zero2.json
257
+ gradient_checkpointing: false
258
+ bf16: true
259
+ tf32: true
260
+ per_device_train_batch_size: 1
261
+ per_device_eval_batch_size: 64
262
+ gradient_accumulation_steps: 1
263
+ dataloader_num_workers: 0
264
+ dataloader_pin_memory: false
265
+ dataloader_persistent_workers: false
266
+ optim: adamw_torch
267
+ adam_beta1: 0.95
268
+ adam_beta2: 0.999
269
+ adam_epsilon: 1.0e-08
270
+ learning_rate: 1.0e-05
271
+ weight_decay: 1.0e-05
272
+ warmup_ratio: 0.05
273
+ lr_scheduler_type: cosine
274
+ logging_steps: 1
275
+ num_train_epochs: 1000
276
+ max_steps: 10
277
+ save_strategy: steps
278
+ save_steps: 10
279
+ save_total_limit: 5
280
+ report_to: none
281
+ seed: 42
282
+ do_eval: false
283
+ ddp_find_unused_parameters: false
284
+ ddp_bucket_cap_mb: 100
285
+ torch_compile_mode: null
286
+ logging_dir: ./checkpoints/wnm_3d_stage3/tensorboard
287
+ backbone_hidden_size: 0
288
+ backbone_cfg:
289
+ _target_: gammanav.vln.model.wnm_3d.backbone.identity.IdentityBackbone
290
+ action_head_cfg:
291
+ config:
292
+ backbone_features_projector_cfg: null
293
+ _target_: gammanav.vln.model.wnm_3d.action_head.wan_flow_matching_action_tf.WANPolicyHeadConfig
294
+ _recursive_: false
295
+ tiled: false
296
+ tile_size_height: 34
297
+ tile_size_width: 34
298
+ tile_stride_height: 18
299
+ tile_stride_width: 16
300
+ lora_rank: 4
301
+ lora_alpha: 4
302
+ num_frames: 33
303
+ num_frame_per_block: 2
304
+ lora_target_modules: q,k,v,o,ffn.0,ffn.2
305
+ init_lora_weights: kaiming
306
+ train_architecture: full
307
+ use_vggt_geometry_adapter: true
308
+ vggt_checkpoint_path: ./checkpoints/vggt_omega_1b_512.pt
309
+ vggt_image_resolution: 512
310
+ vggt_resize_mode: square
311
+ vggt_adapter_dim: 512
312
+ vggt_adapter_blocks: 2
313
+ vggt_adapter_heads: 8
314
+ max_state_dim: 64
315
+ max_action_dim: 32
316
+ diffusion_model_cfg:
317
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_dit_action_casual_chunk.CausalWanModel
318
+ _convert_: object
319
+ diffusion_model_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B
320
+ model_type: ti2v
321
+ frame_seqlen: 50
322
+ dim: 3072
323
+ in_dim: 48
324
+ ffn_dim: 14336
325
+ out_dim: 48
326
+ freq_dim: 256
327
+ eps: 1.0e-06
328
+ num_heads: 24
329
+ num_layers: 30
330
+ max_chunk_size: 4
331
+ num_frame_per_block: 2
332
+ num_action_per_block: 8
333
+ num_state_per_block: 1
334
+ concat_first_frame_latent: false
335
+ text_encoder_cfg:
336
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_text_encoder.WanTextEncoder
337
+ _convert_: object
338
+ text_encoder_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B/models_t5_umt5-xxl-enc-bf16.pth
339
+ image_encoder_cfg:
340
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_image_encoder.WanImageEncoder
341
+ _convert_: object
342
+ image_encoder_pretrained_path: ./checkpoints/Wan2.1-I2V-14B-480P/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth
343
+ vae_cfg:
344
+ _target_: gammanav.vln.model.wnm_3d.modules.wan_video_vae.WanVideoVAE38
345
+ _convert_: object
346
+ vae_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth
347
+ z_dim: 48
348
+ dim: 160
349
+ action_dim: 32
350
+ action_horizon: 8
351
+ num_inference_timesteps: 4
352
+ decouple_video_action_noise: false
353
+ video_noise_beta_alpha: 3.0
354
+ video_noise_beta_beta: 1.0
355
+ tune_projector: true
356
+ tune_diffusion_model: true
357
+ target_video_height: 160
358
+ target_video_width: 320
359
+ _target_: gammanav.vln.model.wnm_3d.action_head.wan_flow_matching_action_tf.WANPolicyHead
360
+ _convert_: object
361
+ dit_version: ./checkpoints/Wan2.2-TI2V-5B
362
+ text_encoder_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B/models_t5_umt5-xxl-enc-bf16.pth
363
+ image_encoder_pretrained_path: ./checkpoints/Wan2.1-I2V-14B-480P/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth
364
+ vae_pretrained_path: ./checkpoints/Wan2.2-TI2V-5B/Wan2.2_VAE.pth
365
+ train_architecture: full
366
+ num_frame_per_block: 2
367
+ num_action_per_block: 8
368
+ num_state_per_block: 1
369
+ frame_seqlen: 50
370
+ use_vggt_geometry_adapter: true
371
+ vggt_omega_checkpoint_path: ./checkpoints/vggt_omega_1b_512.pt
372
+ vggt_image_resolution: 512
373
+ vggt_resize_mode: square
374
+ vggt_adapter_dim: 512
375
+ vggt_adapter_blocks: 2
376
+ vggt_adapter_heads: 8
377
+ max_length: 512
378
+ tokenizer_path: ./checkpoints/umt5-xxl
379
+ data_collator:
380
+ _target_: gammanav.vln.model.wnm_3d.transform.wnm_3d_cotrain.DefaultDataCollator
381
+ tokenizer_path: ./checkpoints/umt5-xxl
382
+ max_length: 512
383
+ num_visual_tokens_per_frame: 16
384
+ max_state_dim: 64
385
+ max_action_dim: 32
386
+ language_dropout_prob: 0.0
387
+ model_specific_transform:
388
+ _target_: gammanav.vln.model.wnm_3d.transform.wnm_3d_cotrain.WNM3DTransform
389
+ default_instruction: Perform the default behavior.
390
+ language_dropout_prob: 0.0
391
+ always_use_default_instruction: false
392
+ max_state_dim: 64
393
+ max_action_dim: 32
394
+ max_length: 512
395
+ state_horizon: 4
396
+ action_horizon: 8
397
+ tokenizer_path: ./checkpoints/umt5-xxl
398
+ image_resolution_width: 320
399
+ image_resolution_height: 160
400
+ num_frames: 33
401
+ state_horizon: 4
402
+ action_horizon: 8
403
+ vae_temporal_stride: 4
404
+ max_chunk_size: 4
405
+ video_frame_stride: 1
406
+ action_frame_stride: 1
407
+ history_sampling: uniform
408
+ interiorgs_data_root: ./data/interiorgs_lerobot_seen
409
+ modality_config_interiorgs:
410
+ video:
411
+ _target_: gammanav.vln.data.dataset.ModalityConfig
412
+ delta_indices: []
413
+ eval_delta_indices:
414
+ - 0
415
+ modality_keys:
416
+ - video.rgb
417
+ state:
418
+ _target_: gammanav.vln.data.dataset.ModalityConfig
419
+ delta_indices: []
420
+ modality_keys:
421
+ - state.nav_pose
422
+ action:
423
+ _target_: gammanav.vln.data.dataset.ModalityConfig
424
+ delta_indices: []
425
+ modality_keys:
426
+ - action.nav_delta
427
+ language:
428
+ _target_: gammanav.vln.data.dataset.ModalityConfig
429
+ delta_indices:
430
+ - 0
431
+ modality_keys:
432
+ - annotation.language.language_instruction
433
+ transform_interiorgs:
434
+ _target_: gammanav.vln.data.transform.ComposedModalityTransform
435
+ transforms:
436
+ - _target_: gammanav.vln.data.transform.VideoToTensor
437
+ apply_to:
438
+ - video.rgb
439
+ - _target_: gammanav.vln.data.transform.VideoCrop
440
+ apply_to:
441
+ - video.rgb
442
+ scale: 0.95
443
+ mode: random
444
+ - _target_: gammanav.vln.data.transform.VideoResize
445
+ apply_to:
446
+ - video.rgb
447
+ height: 160
448
+ width: 320
449
+ interpolation: linear
450
+ - _target_: gammanav.vln.data.transform.VideoColorJitter
451
+ apply_to:
452
+ - video.rgb
453
+ brightness: 0.3
454
+ contrast: 0.4
455
+ saturation: 0.5
456
+ hue: 0.08
457
+ - _target_: gammanav.vln.data.transform.VideoToNumpy
458
+ apply_to:
459
+ - video.rgb
460
+ - _target_: gammanav.vln.data.transform.StateActionToTensor
461
+ apply_to:
462
+ - state.nav_pose
463
+ - _target_: gammanav.vln.data.transform.StateActionTransform
464
+ apply_to:
465
+ - state.nav_pose
466
+ normalization_modes:
467
+ state.nav_pose: q99
468
+ - _target_: gammanav.vln.data.transform.StateActionToTensor
469
+ apply_to:
470
+ - action.nav_delta
471
+ - _target_: gammanav.vln.data.transform.StateActionTransform
472
+ apply_to:
473
+ - action.nav_delta
474
+ normalization_modes:
475
+ action.nav_delta: q99
476
+ - _target_: gammanav.vln.data.transform.ConcatTransform
477
+ video_concat_order:
478
+ - video.rgb
479
+ state_concat_order:
480
+ - state.nav_pose
481
+ action_concat_order:
482
+ - action.nav_delta
483
+ - _target_: gammanav.vln.model.wnm_3d.transform.wnm_3d_cotrain.WNM3DTransform
484
+ default_instruction: Perform the default behavior.
485
+ language_dropout_prob: 0.0
486
+ always_use_default_instruction: false
487
+ max_state_dim: 64
488
+ max_action_dim: 32
489
+ max_length: 512
490
+ state_horizon: 4
491
+ action_horizon: 8
492
+ tokenizer_path: ./checkpoints/umt5-xxl
493
+ modality_configs:
494
+ interiorgs:
495
+ video:
496
+ _target_: gammanav.vln.data.dataset.ModalityConfig
497
+ delta_indices: []
498
+ eval_delta_indices:
499
+ - 0
500
+ modality_keys:
501
+ - video.rgb
502
+ state:
503
+ _target_: gammanav.vln.data.dataset.ModalityConfig
504
+ delta_indices: []
505
+ modality_keys:
506
+ - state.nav_pose
507
+ action:
508
+ _target_: gammanav.vln.data.dataset.ModalityConfig
509
+ delta_indices: []
510
+ modality_keys:
511
+ - action.nav_delta
512
+ language:
513
+ _target_: gammanav.vln.data.dataset.ModalityConfig
514
+ delta_indices:
515
+ - 0
516
+ modality_keys:
517
+ - annotation.language.language_instruction
518
+ transforms:
519
+ interiorgs:
520
+ _target_: gammanav.vln.data.transform.ComposedModalityTransform
521
+ transforms:
522
+ - _target_: gammanav.vln.data.transform.VideoToTensor
523
+ apply_to:
524
+ - video.rgb
525
+ - _target_: gammanav.vln.data.transform.VideoCrop
526
+ apply_to:
527
+ - video.rgb
528
+ scale: 0.95
529
+ mode: random
530
+ - _target_: gammanav.vln.data.transform.VideoResize
531
+ apply_to:
532
+ - video.rgb
533
+ height: 160
534
+ width: 320
535
+ interpolation: linear
536
+ - _target_: gammanav.vln.data.transform.VideoColorJitter
537
+ apply_to:
538
+ - video.rgb
539
+ brightness: 0.3
540
+ contrast: 0.4
541
+ saturation: 0.5
542
+ hue: 0.08
543
+ - _target_: gammanav.vln.data.transform.VideoToNumpy
544
+ apply_to:
545
+ - video.rgb
546
+ - _target_: gammanav.vln.data.transform.StateActionToTensor
547
+ apply_to:
548
+ - state.nav_pose
549
+ - _target_: gammanav.vln.data.transform.StateActionTransform
550
+ apply_to:
551
+ - state.nav_pose
552
+ normalization_modes:
553
+ state.nav_pose: q99
554
+ - _target_: gammanav.vln.data.transform.StateActionToTensor
555
+ apply_to:
556
+ - action.nav_delta
557
+ - _target_: gammanav.vln.data.transform.StateActionTransform
558
+ apply_to:
559
+ - action.nav_delta
560
+ normalization_modes:
561
+ action.nav_delta: q99
562
+ - _target_: gammanav.vln.data.transform.ConcatTransform
563
+ video_concat_order:
564
+ - video.rgb
565
+ state_concat_order:
566
+ - state.nav_pose
567
+ action_concat_order:
568
+ - action.nav_delta
569
+ - _target_: gammanav.vln.model.wnm_3d.transform.wnm_3d_cotrain.WNM3DTransform
570
+ default_instruction: Perform the default behavior.
571
+ language_dropout_prob: 0.0
572
+ always_use_default_instruction: false
573
+ max_state_dim: 64
574
+ max_action_dim: 32
575
+ max_length: 512
576
+ state_horizon: 4
577
+ action_horizon: 8
578
+ tokenizer_path: ./checkpoints/umt5-xxl
579
+ mixture_dataset_cls: gammanav.vln.data.dataset.lerobot.LeRobotMixtureDataset.from_mixture_spec
580
+ single_dataset_cls: gammanav.vln.data.dataset.lerobot_sharded.ShardedLeRobotSingleActionChunkDatasetInteriorGS
581
+ total_training_steps: 89725000
wnm_3d_stage3/experiment_cfg/metadata.json ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "interiorgs": {
3
+ "statistics": {
4
+ "state": {
5
+ "nav_pose": {
6
+ "max": [
7
+ 0.0,
8
+ 0.0,
9
+ 0.0
10
+ ],
11
+ "min": [
12
+ 0.0,
13
+ 0.0,
14
+ 0.0
15
+ ],
16
+ "mean": [
17
+ 0.0,
18
+ 0.0,
19
+ 0.0
20
+ ],
21
+ "std": [
22
+ 0.0,
23
+ 0.0,
24
+ 0.0
25
+ ],
26
+ "q01": [
27
+ -1.0,
28
+ -1.0,
29
+ -1.0
30
+ ],
31
+ "q99": [
32
+ 1.0,
33
+ 1.0,
34
+ 1.0
35
+ ]
36
+ }
37
+ },
38
+ "action": {
39
+ "nav_delta": {
40
+ "max": [
41
+ 4.734772682189941,
42
+ 2.703505516052246,
43
+ 12.561675071716309
44
+ ],
45
+ "min": [
46
+ -4.257997035980225,
47
+ -3.7617416381835938,
48
+ -12.553963661193848
49
+ ],
50
+ "mean": [
51
+ 0.24795560538768768,
52
+ 0.006410803645849228,
53
+ 0.017522765323519707
54
+ ],
55
+ "std": [
56
+ 0.3869488537311554,
57
+ 0.3040892779827118,
58
+ 0.7763124704360962
59
+ ],
60
+ "q01": [
61
+ -0.6557483077049255,
62
+ -0.8707880973815918,
63
+ -1.104737639427185
64
+ ],
65
+ "q99": [
66
+ 1.4973504543304443,
67
+ 0.861659824848175,
68
+ 1.2756091356277466
69
+ ]
70
+ }
71
+ }
72
+ },
73
+ "modalities": {
74
+ "video": {
75
+ "rgb": {
76
+ "resolution": [
77
+ 480,
78
+ 360
79
+ ],
80
+ "channels": 3,
81
+ "fps": 3.3333333333333335
82
+ }
83
+ },
84
+ "state": {
85
+ "nav_pose": {
86
+ "absolute": false,
87
+ "rotation_type": null,
88
+ "shape": [
89
+ 3
90
+ ],
91
+ "continuous": true
92
+ }
93
+ },
94
+ "action": {
95
+ "nav_delta": {
96
+ "absolute": false,
97
+ "rotation_type": null,
98
+ "shape": [
99
+ 3
100
+ ],
101
+ "continuous": true
102
+ }
103
+ }
104
+ },
105
+ "embodiment_tag": "interiorgs"
106
+ }
107
+ }
wnm_3d_stage3/model-00001-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0e0ff763ed8bb6b855782e10f1577d676bc7abce2df0e46cea5d340a05963a5
3
+ size 4935816720
wnm_3d_stage3/model-00002-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3dd6a6ed2735b920badc8b58864ac567f921b5e1ab537f9c755e35e6c65ebd8c
3
+ size 4983107552
wnm_3d_stage3/model-00003-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9d82ec1b43fcb2a8ce39fe692993c8d5944a7620f086b61257b2d79dbf69b156
3
+ size 4919126194
wnm_3d_stage3/model-00004-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:936913a457e1d5bce09acc2b5044db1d34524698cc5a838282d1d196875a326c
3
+ size 4997878856
wnm_3d_stage3/model-00005-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d7dffc1c6e31c343e991d82062bfd7c27b120bed9139c832354621042933d508
3
+ size 4947510648
wnm_3d_stage3/model-00006-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4b3ca202db5df95e571095844f5073a47e761cb94ad7f5be8c237d67c32b3944
3
+ size 2329945620
wnm_3d_stage3/model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff