| recon: |
| spec: |
| _target_: far.builders.data.build_dataspec |
| local_pos_stats: |
| min: |
| - -2.5 |
| - -4 |
| max: |
| - 5 |
| - 4 |
| meters_per_waypoint: 0.25 |
| max_frame_offset: 128 |
| len_traj_pred: ${len_traj_pred} |
| planner_mu_init: |
| - -0.1 |
| - 0 |
| - 0 |
| planner_sigma_init: |
| - 0.02 |
| - 0.1 |
| - 0.3142 |
| pose_fields: |
| - x |
| - 'y' |
| - z |
| - pitch |
| - yaw |
| pose_dim: 5 |
| pos_dim: 2 |
| angle_dim: 1 |
| pose_mode: 2d |
| fps: 4.0 |
| shared: |
| manifest_path: results/manifests/recon/recon_split.json |
| backend: |
| _target_: far.data.backends.NavigationBackend |
| transform: |
| _target_: far.data.transforms.make_centercrop_resize |
| image_height: ${image_height} |
| image_width: ${image_width} |
| sampler: |
| context_goal: |
| _target_: far.data.samplers.ContextGoalSampler |
| context_pool_size: 12 |
| len_traj_pred: ${len_traj_pred} |
| goals_per_obs: ${goals_per_obs} |
| context_traj: |
| _target_: far.data.samplers.ContextTrajectorySampler |
| context_pool_size: 12 |
| len_traj_pred: ${len_traj_pred} |
| formatter: |
| context_pose: |
| _target_: far.data.formatters.ContextPoseFormatter2D |
| spec: ${recon.spec} |
| train: |
| _target_: far.data.datasets.FormattedVideoDataset |
| formatter: ${recon.formatter.context_pose} |
| base_dataset: |
| _target_: far.data.datasets.VideoDataset |
| manifest_path: ${recon.shared.manifest_path} |
| index_path: results/indices/recon/recon_train.json |
| backend: ${recon.shared.backend} |
| sampler: ${recon.sampler.context_goal} |
| transform: ${recon.shared.transform} |
| test: |
| _target_: far.data.datasets.FormattedVideoDataset |
| formatter: ${recon.formatter.context_pose} |
| base_dataset: |
| _target_: far.data.datasets.VideoDataset |
| manifest_path: ${recon.shared.manifest_path} |
| index_path: results/indices/recon/recon_test.json |
| backend: ${recon.shared.backend} |
| sampler: ${recon.sampler.context_goal} |
| transform: ${recon.shared.transform} |
| eval: |
| _target_: far.data.datasets.FormattedVideoDataset |
| formatter: ${recon.formatter.context_pose} |
| base_dataset: |
| _target_: far.data.datasets.VideoDataset |
| manifest_path: ${recon.shared.manifest_path} |
| index_path: results/indices/recon/recon_test_time.json |
| backend: ${recon.shared.backend} |
| sampler: ${recon.sampler.context_traj} |
| transform: ${recon.shared.transform} |
| eval_traj: |
| _target_: far.data.datasets.FormattedVideoDataset |
| formatter: ${recon.formatter.context_pose} |
| base_dataset: |
| _target_: far.data.datasets.VideoDataset |
| manifest_path: ${recon.shared.manifest_path} |
| index_path: results/indices/recon/recon_test_plan.json |
| backend: ${recon.shared.backend} |
| sampler: ${recon.sampler.context_traj} |
| transform: ${recon.shared.transform} |
| loopnav: |
| spec: |
| _target_: far.builders.data.build_dataspec |
| local_pos_stats: |
| min: |
| - -2.5 |
| - -4 |
| - 0 |
| max: |
| - 5 |
| - 4 |
| - 1 |
| meters_per_waypoint: 0.25 |
| max_frame_offset: 128 |
| len_traj_pred: ${len_traj_pred} |
| planner_mu_init: |
| - -0.1 |
| - 0 |
| - 0 |
| - 0 |
| - 0 |
| planner_sigma_init: |
| - 0.02 |
| - 0.1 |
| - 0 |
| - 0 |
| - 0.3142 |
| pose_fields: |
| - x |
| - 'y' |
| - z |
| - pitch |
| - yaw |
| pose_dim: 5 |
| pos_dim: 3 |
| angle_dim: 2 |
| pose_mode: 3d |
| fps: 20.0 |
| shared: |
| manifest_path: results/manifests/loopnav/loopnav_split.json |
| backend: |
| _target_: far.data.backends.LoopNavBackend |
| standard_frame: false |
| transform: |
| _target_: far.data.transforms.make_resize |
| image_height: ${image_height} |
| image_width: ${image_width} |
| sampler: |
| random_clip: |
| _target_: far.data.samplers.RandomClipSampler |
| clip_len: 16 |
| frame_stride: 10 |
| context_goal: |
| _target_: far.data.samplers.ContextGoalSampler |
| context_pool_size: 100 |
| len_traj_pred: ${len_traj_pred} |
| goals_per_obs: ${goals_per_obs} |
| context_traj: |
| _target_: far.data.samplers.ContextTrajectorySampler |
| context_pool_size: 100 |
| len_traj_pred: ${len_traj_pred} |
| formatter: |
| frame: |
| _target_: far.data.formatters.FrameFormatter |
| context_pose: |
| _target_: far.data.formatters.ContextPoseFormatter |
| spec: ${loopnav.spec} |
| train_tokenizer: |
| _target_: far.data.datasets.FormattedIterableDataset |
| formatter: ${loopnav.formatter.frame} |
| base_dataset: |
| _target_: far.data.datasets.WebVideoIterableDataset |
| manifest_path: results/manifests/loopnav/loopnav.json |
| index_path: results/indices/loopnav/loopnav_train_ABCA.json |
| backend: |
| _target_: far.data.backends.WebDatasetBackend |
| sampler: ${loopnav.sampler.random_clip} |
| transform: ${loopnav.shared.transform} |
| cache_dir: null |
| cache_size: 100 |
| shuffle_shards: 32 |
| shuffle_samples: 100 |
| loopnav_latent: |
| spec: |
| _target_: far.builders.data.build_dataspec |
| local_pos_stats: |
| min: |
| - 0 |
| - 0 |
| - 0 |
| max: |
| - 1 |
| - 1 |
| - 1 |
| meters_per_waypoint: 1.0 |
| max_frame_offset: 20.0 |
| len_traj_pred: ${len_traj_pred} |
| planner_mu_init: |
| - -0.1 |
| - 0 |
| - 0 |
| - 0 |
| - 0 |
| planner_sigma_init: |
| - 0.02 |
| - 0.1 |
| - 0 |
| - 0 |
| - 0.3142 |
| pose_fields: |
| - x |
| - 'y' |
| - z |
| - pitch |
| - yaw |
| pose_dim: 5 |
| pos_dim: 3 |
| angle_dim: 2 |
| pose_mode: 3d |
| fps: 20.0 |
| shared: |
| manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/loopnav/loopnav_latent.json |
| backend: |
| _target_: far.data.backends.LatentLoopNavBackend |
| key_suffix: .keys_jepa.npy |
| standard_frame: true |
| transform: null |
| sampler: |
| random_clip: |
| _target_: far.data.samplers.RandomClipSampler |
| clip_len: 16 |
| frame_stride: 10 |
| context_goal: |
| _target_: far.data.samplers.ContextGoalSampler |
| context_pool_size: 200 |
| len_traj_pred: ${len_traj_pred} |
| goals_per_obs: ${goals_per_obs} |
| context_goal_test: |
| _target_: far.data.samplers.ContextGoalSampler |
| context_pool_size: 200 |
| len_traj_pred: ${len_traj_pred} |
| goals_per_obs: 4 |
| context_traj: |
| _target_: far.data.samplers.ContextTrajectorySampler |
| context_pool_size: 200 |
| len_traj_pred: ${len_traj_pred} |
| formatter: |
| frame: |
| _target_: far.data.formatters.FrameFormatter |
| context_pose: |
| _target_: far.data.formatters.ContextPoseFormatter |
| spec: ${loopnav_latent.spec} |
| train: |
| _target_: far.data.datasets.FormattedVideoDataset |
| formatter: ${loopnav_latent.formatter.context_pose} |
| base_dataset: |
| _target_: far.data.datasets.VideoDataset |
| manifest_path: ${loopnav_latent.shared.manifest_path} |
| index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_train.json |
| backend: ${loopnav_latent.shared.backend} |
| sampler: ${loopnav_latent.sampler.context_goal} |
| transform: ${loopnav_latent.shared.transform} |
| test: |
| _target_: far.data.datasets.FormattedVideoDataset |
| formatter: ${loopnav_latent.formatter.context_pose} |
| base_dataset: |
| _target_: far.data.datasets.VideoDataset |
| manifest_path: ${loopnav_latent.shared.manifest_path} |
| index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_test.json |
| backend: ${loopnav_latent.shared.backend} |
| sampler: ${loopnav_latent.sampler.context_goal_test} |
| transform: ${loopnav_latent.shared.transform} |
| model: |
| tokenizer: |
| _target_: far.tokenizers.vit_vae.ViTVAE |
| kl_weight: 1.0e-06 |
| lpips_weight: 0.0 |
| ckpt_path: ${oc.env:FAR_MODELS_ROOT}/oasis_500m_vit_vae.pth |
| encoder: |
| _target_: far.tokenizers.vit_vae.Encoder |
| input_height: ${image_height} |
| input_width: ${image_width} |
| patch_size: 20 |
| dim: 1024 |
| depth: 6 |
| heads: 16 |
| latent_dim: 16 |
| use_variational: true |
| mlp_ratio: 4.0 |
| decoder: |
| _target_: far.tokenizers.vit_vae.Decoder |
| input_height: ${image_height} |
| input_width: ${image_width} |
| patch_size: 20 |
| dim: 1024 |
| depth: 12 |
| heads: 16 |
| latent_dim: 16 |
| mlp_ratio: 4.0 |
| generator: |
| _target_: far.generators.cdit_mod_pose_fa.CDiT |
| depth: 12 |
| hidden_size: 768 |
| patch_size: 2 |
| num_heads: 12 |
| context_size: 4 |
| ckpt_path: null |
| action_dim: ${action_dim} |
| pose_dim: ${pose_dim} |
| fov_half_h: 52.5 |
| fov_half_v: 37.5 |
| local_pos_stats: ${loopnav_latent.spec.local_pos_stats} |
| stride_to_seconds: 1.0 |
| memory: |
| _target_: far.memories.pnp_cl.PnPRetrieval |
| ckpt_path: null |
| chunk_size: 10 |
| num_sets: 5 |
| softmax_temperature: 1.0 |
| random_replace_prob: 0.0 |
| random_replace_warmup_prob: 0.0 |
| random_replace_warmup_steps: 50000 |
| query_adapter_hidden: null |
| query_vit: |
| _target_: far.memories.pnp_cl.QueryViT |
| img_height: 18 |
| img_width: 32 |
| patch_size: 2 |
| in_chans: 16 |
| embed_dim: 384 |
| depth: 6 |
| num_heads: 12 |
| action_dim: ${action_dim} |
| key_dim: 256 |
| mlp_ratio: 4.0 |
| metadata_scorer: false |
| metadata_hidden: 128 |
| local_pos_stats: ${loopnav_latent.spec.local_pos_stats} |
| stride_to_seconds: 1.0 |
| freeze_encoder: true |
| _target_: far.builders.models.build_ldwm |
| _recursive_: false |
| condition_on_context_time: true |
| condition_on_context_pose: true |
| retriever_loss_weight: 0.0 |
| diffusion: |
| _target_: far.generators.diffusion.create_diffusion |
| timestep_respacing: null |
| noise_schedule: linear |
| diffusion_steps: 1000 |
| learn_sigma: true |
| predict_xstart: false |
| sigma_small: false |
| use_kl: false |
| rescale_learned_sigmas: false |
| wandb: |
| project: PMem-World |
| name: null |
| entity: null |
| mode: online |
| group: loopnav_latent |
| tags: |
| - loopnav_latent |
| run_name: nwm_cdit_b_pnp_jepa_posefa_l0_ddim20 |
| context_size: 4 |
| action_dim: 5 |
| pose_dim: 5 |
| image_height: 360 |
| image_width: 640 |
| len_traj_pred: 128 |
| goals_per_obs: 4 |
| lr: 0.0001 |
| lr_mem: 0.0001 |
| weight_decay: 0.0 |
| grad_clip_val: 1.0 |
| iterations: 5000000 |
| warmup_steps: 5000 |
| from_checkpoint: null |
| global_seed: 0 |
| log_every: 100 |
| ckpt_every: 100000 |
| eval_every: 2500 |
| rag_every: 20 |
| rag_noise_levels: 4 |
| local_rag: true |
| local_rag_weight: 1.0 |
| bfloat16: false |
| torch_compile: true |
| batch_size: 32 |
| eval_batch_size: 64 |
| num_workers: 12 |
| datasets: |
| - loopnav_latent |
|
|
