recon:
  spec:
    _target_: far.builders.data.build_dataspec
    local_pos_stats:
      min:
      - -2.5
      - -4
      max:
      - 5
      - 4
    meters_per_waypoint: 0.25
    max_frame_offset: 128
    len_traj_pred: ${len_traj_pred}
    planner_mu_init:
    - -0.1
    - 0
    - 0
    planner_sigma_init:
    - 0.02
    - 0.1
    - 0.3142
    pose_fields:
    - x
    - 'y'
    - z
    - pitch
    - yaw
    pose_dim: 5
    pos_dim: 2
    angle_dim: 1
    pose_mode: 2d
    fps: 4.0
  shared:
    manifest_path: results/manifests/recon/recon_split.json
    backend:
      _target_: far.data.backends.NavigationBackend
    transform:
      _target_: far.data.transforms.make_centercrop_resize
      image_height: ${image_height}
      image_width: ${image_width}
  sampler:
    context_goal:
      _target_: far.data.samplers.ContextGoalSampler
      context_pool_size: 12
      len_traj_pred: ${len_traj_pred}
      goals_per_obs: ${goals_per_obs}
    context_traj:
      _target_: far.data.samplers.ContextTrajectorySampler
      context_pool_size: 12
      len_traj_pred: ${len_traj_pred}
  formatter:
    context_pose:
      _target_: far.data.formatters.ContextPoseFormatter2D
      spec: ${recon.spec}
  train:
    _target_: far.data.datasets.FormattedVideoDataset
    formatter: ${recon.formatter.context_pose}
    base_dataset:
      _target_: far.data.datasets.VideoDataset
      manifest_path: ${recon.shared.manifest_path}
      index_path: results/indices/recon/recon_train.json
      backend: ${recon.shared.backend}
      sampler: ${recon.sampler.context_goal}
      transform: ${recon.shared.transform}
  test:
    _target_: far.data.datasets.FormattedVideoDataset
    formatter: ${recon.formatter.context_pose}
    base_dataset:
      _target_: far.data.datasets.VideoDataset
      manifest_path: ${recon.shared.manifest_path}
      index_path: results/indices/recon/recon_test.json
      backend: ${recon.shared.backend}
      sampler: ${recon.sampler.context_goal}
      transform: ${recon.shared.transform}
  eval:
    _target_: far.data.datasets.FormattedVideoDataset
    formatter: ${recon.formatter.context_pose}
    base_dataset:
      _target_: far.data.datasets.VideoDataset
      manifest_path: ${recon.shared.manifest_path}
      index_path: results/indices/recon/recon_test_time.json
      backend: ${recon.shared.backend}
      sampler: ${recon.sampler.context_traj}
      transform: ${recon.shared.transform}
  eval_traj:
    _target_: far.data.datasets.FormattedVideoDataset
    formatter: ${recon.formatter.context_pose}
    base_dataset:
      _target_: far.data.datasets.VideoDataset
      manifest_path: ${recon.shared.manifest_path}
      index_path: results/indices/recon/recon_test_plan.json
      backend: ${recon.shared.backend}
      sampler: ${recon.sampler.context_traj}
      transform: ${recon.shared.transform}
loopnav:
  spec:
    _target_: far.builders.data.build_dataspec
    local_pos_stats:
      min:
      - -2.5
      - -4
      - 0
      max:
      - 5
      - 4
      - 1
    meters_per_waypoint: 0.25
    max_frame_offset: 128
    len_traj_pred: ${len_traj_pred}
    planner_mu_init:
    - -0.1
    - 0
    - 0
    - 0
    - 0
    planner_sigma_init:
    - 0.02
    - 0.1
    - 0
    - 0
    - 0.3142
    pose_fields:
    - x
    - 'y'
    - z
    - pitch
    - yaw
    pose_dim: 5
    pos_dim: 3
    angle_dim: 2
    pose_mode: 3d
    fps: 20.0
  shared:
    manifest_path: results/manifests/loopnav/loopnav_split.json
    backend:
      _target_: far.data.backends.LoopNavBackend
      standard_frame: false
    transform:
      _target_: far.data.transforms.make_resize
      image_height: ${image_height}
      image_width: ${image_width}
  sampler:
    random_clip:
      _target_: far.data.samplers.RandomClipSampler
      clip_len: 16
      frame_stride: 10
    context_goal:
      _target_: far.data.samplers.ContextGoalSampler
      context_pool_size: 100
      len_traj_pred: ${len_traj_pred}
      goals_per_obs: ${goals_per_obs}
    context_traj:
      _target_: far.data.samplers.ContextTrajectorySampler
      context_pool_size: 100
      len_traj_pred: ${len_traj_pred}
  formatter:
    frame:
      _target_: far.data.formatters.FrameFormatter
    context_pose:
      _target_: far.data.formatters.ContextPoseFormatter
      spec: ${loopnav.spec}
  train_tokenizer:
    _target_: far.data.datasets.FormattedIterableDataset
    formatter: ${loopnav.formatter.frame}
    base_dataset:
      _target_: far.data.datasets.WebVideoIterableDataset
      manifest_path: results/manifests/loopnav/loopnav.json
      index_path: results/indices/loopnav/loopnav_train_ABCA.json
      backend:
        _target_: far.data.backends.WebDatasetBackend
      sampler: ${loopnav.sampler.random_clip}
      transform: ${loopnav.shared.transform}
      cache_dir: null
      cache_size: 100
      shuffle_shards: 32
      shuffle_samples: 100
loopnav_latent:
  spec:
    _target_: far.builders.data.build_dataspec
    local_pos_stats:
      min:
      - 0
      - 0
      - 0
      max:
      - 1
      - 1
      - 1
    meters_per_waypoint: 1.0
    max_frame_offset: 20.0
    len_traj_pred: ${len_traj_pred}
    planner_mu_init:
    - -0.1
    - 0
    - 0
    - 0
    - 0
    planner_sigma_init:
    - 0.02
    - 0.1
    - 0
    - 0
    - 0.3142
    pose_fields:
    - x
    - 'y'
    - z
    - pitch
    - yaw
    pose_dim: 5
    pos_dim: 3
    angle_dim: 2
    pose_mode: 3d
    fps: 20.0
  shared:
    manifest_path: ${oc.env:FAR_RESULT_ROOT}/manifests/loopnav/loopnav_latent.json
    backend:
      _target_: far.data.backends.LatentLoopNavBackend
      key_suffix: .keys_jepa.npy
      standard_frame: true
    transform: null
  sampler:
    random_clip:
      _target_: far.data.samplers.RandomClipSampler
      clip_len: 16
      frame_stride: 10
    context_goal:
      _target_: far.data.samplers.ContextGoalSampler
      context_pool_size: 200
      len_traj_pred: ${len_traj_pred}
      goals_per_obs: ${goals_per_obs}
    context_goal_test:
      _target_: far.data.samplers.ContextGoalSampler
      context_pool_size: 200
      len_traj_pred: ${len_traj_pred}
      goals_per_obs: 4
    context_traj:
      _target_: far.data.samplers.ContextTrajectorySampler
      context_pool_size: 200
      len_traj_pred: ${len_traj_pred}
  formatter:
    frame:
      _target_: far.data.formatters.FrameFormatter
    context_pose:
      _target_: far.data.formatters.ContextPoseFormatter
      spec: ${loopnav_latent.spec}
  train:
    _target_: far.data.datasets.FormattedVideoDataset
    formatter: ${loopnav_latent.formatter.context_pose}
    base_dataset:
      _target_: far.data.datasets.VideoDataset
      manifest_path: ${loopnav_latent.shared.manifest_path}
      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_train.json
      backend: ${loopnav_latent.shared.backend}
      sampler: ${loopnav_latent.sampler.context_goal}
      transform: ${loopnav_latent.shared.transform}
  test:
    _target_: far.data.datasets.FormattedVideoDataset
    formatter: ${loopnav_latent.formatter.context_pose}
    base_dataset:
      _target_: far.data.datasets.VideoDataset
      manifest_path: ${loopnav_latent.shared.manifest_path}
      index_path: ${oc.env:FAR_RESULT_ROOT}/indices/loopnav/loopnav_latent_test.json
      backend: ${loopnav_latent.shared.backend}
      sampler: ${loopnav_latent.sampler.context_goal_test}
      transform: ${loopnav_latent.shared.transform}
model:
  tokenizer:
    _target_: far.tokenizers.vit_vae.ViTVAE
    kl_weight: 1.0e-06
    lpips_weight: 0.0
    ckpt_path: ${oc.env:FAR_MODELS_ROOT}/oasis_500m_vit_vae.pth
    encoder:
      _target_: far.tokenizers.vit_vae.Encoder
      input_height: ${image_height}
      input_width: ${image_width}
      patch_size: 20
      dim: 1024
      depth: 6
      heads: 16
      latent_dim: 16
      use_variational: true
      mlp_ratio: 4.0
    decoder:
      _target_: far.tokenizers.vit_vae.Decoder
      input_height: ${image_height}
      input_width: ${image_width}
      patch_size: 20
      dim: 1024
      depth: 12
      heads: 16
      latent_dim: 16
      mlp_ratio: 4.0
  generator:
    _target_: far.generators.cdit_mod_pose_fa.CDiT
    depth: 12
    hidden_size: 768
    patch_size: 2
    num_heads: 12
    context_size: 4
    ckpt_path: null
    action_dim: ${action_dim}
    pose_dim: ${pose_dim}
    fov_half_h: 52.5
    fov_half_v: 37.5
    local_pos_stats: ${loopnav_latent.spec.local_pos_stats}
    stride_to_seconds: 1.0
  memory:
    _target_: far.memories.pnp_cl.PnPRetrieval
    ckpt_path: null
    chunk_size: 10
    num_sets: 5
    softmax_temperature: 1.0
    random_replace_prob: 0.0
    random_replace_warmup_prob: 0.0
    random_replace_warmup_steps: 50000
    query_adapter_hidden: null
    query_vit:
      _target_: far.memories.pnp_cl.QueryViT
      img_height: 18
      img_width: 32
      patch_size: 2
      in_chans: 16
      embed_dim: 384
      depth: 6
      num_heads: 12
      action_dim: ${action_dim}
      key_dim: 256
      mlp_ratio: 4.0
    metadata_scorer: false
    metadata_hidden: 128
    local_pos_stats: ${loopnav_latent.spec.local_pos_stats}
    stride_to_seconds: 1.0
    freeze_encoder: true
  _target_: far.builders.models.build_ldwm
  _recursive_: false
  condition_on_context_time: true
  condition_on_context_pose: true
  retriever_loss_weight: 0.0
  diffusion:
    _target_: far.generators.diffusion.create_diffusion
    timestep_respacing: null
    noise_schedule: linear
    diffusion_steps: 1000
    learn_sigma: true
    predict_xstart: false
    sigma_small: false
    use_kl: false
    rescale_learned_sigmas: false
wandb:
  project: PMem-World
  name: null
  entity: null
  mode: online
  group: loopnav_latent
  tags:
  - loopnav_latent
run_name: nwm_cdit_b_pnp_jepa_posefa_l0_ddim20
context_size: 4
action_dim: 5
pose_dim: 5
image_height: 360
image_width: 640
len_traj_pred: 128
goals_per_obs: 4
lr: 0.0001
lr_mem: 0.0001
weight_decay: 0.0
grad_clip_val: 1.0
iterations: 5000000
warmup_steps: 5000
from_checkpoint: null
global_seed: 0
log_every: 100
ckpt_every: 100000
eval_every: 2500
rag_every: 20
rag_noise_levels: 4
local_rag: true
local_rag_weight: 1.0
bfloat16: false
torch_compile: true
batch_size: 32
eval_batch_size: 64
num_workers: 12
datasets:
- loopnav_latent