Upload 4 files

hierarchos.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:396787af4a7811e5fc87286c9e2f93db37c50b7accbfc72aa45a9e3a08b9a751
+size 931161249

hierarchos_config.json

@@ -0,0 +1,180 @@
+{
+  "mode": "train",
+  "train": null,
+  "hf_dataset": "netcat420/Experiment_0.1",
+  "hf_dataset_config": null,
+  "hf_dataset_split": "train",
+  "text_column": null,
+  "prompt_column": null,
+  "completion_column": null,
+  "model_path": null,
+  "out_dir": "./chatHRM",
+  "tokenizer_path": "openai-community/gpt2",
+  "resume_from_ckpt": null,
+  "shadow_model_path": null,
+  "pre_chunked_dataset": false,
+  "pre_pt_dataset": false,
+  "context_dim": 448,
+  "persistent_dim": 128,
+  "ltm_slots": 1024,
+  "ltm_key_dim": 128,
+  "ltm_val_dim": 128,
+  "h_hidden": 448,
+  "l_hidden": 448,
+  "h_stride": 4,
+  "max_h_steps": 5,
+  "max_l_steps": 5,
+  "ltm_topk": 4,
+  "max_length": 8880,
+  "auto_max_length": false,
+  "use_deepembed": true,
+  "use_rosa": true,
+  "rosa_max_context": 512,
+  "rwkv_head_size": 64,
+  "epochs": 10,
+  "batch_size": 64,
+  "accumulation_steps": 1,
+  "starting_lr": 9e-05,
+  "min_lr": 1e-09,
+  "warmup_steps": 0,
+  "warmup_ratio": 0.03,
+  "disable_lr_schedule": false,
+  "ltm_lr": 1e-08,
+  "min_ltm_lr": 1e-08,
+  "disable_ltm_lr_schedule": true,
+  "rwkv_weight_decay": 0.1,
+  "ltm_score_grad_scale": 1.0,
+  "kayla": false,
+  "alpaca": true,
+  "train_prompt_tokens": true,
+  "strict_all_token_loss": true,
+  "prompt_loss_weight": 0.1,
+  "response_loss_weight": 1.0,
+  "response_boundary_loss_weight": 2.0,
+  "response_boundary_tokens": 64,
+  "min_response_tokens": 128,
+  "drop_empty_completions": true,
+  "assistant_recovery": true,
+  "lora_r": 8,
+  "lora_alpha": 16,
+  "grad_clip": 1.0,
+  "startup_weight_max_abs": 100.0,
+  "max_ce_loss_for_backward": 0.0,
+  "max_commitment_cost_for_backward": 4.0,
+  "max_ponder_cost_for_backward": 0.0,
+  "halt_logit_clamp": 30.0,
+  "recurrent_state_clamp": 50.0,
+  "context_state_clamp": 50.0,
+  "drift_state_clamp": 2.0,
+  "activation_clamp": 100.0,
+  "ponder_loss_weight": 0.003,
+  "commitment_loss_weight": 0.5,
+  "commitment_threshold": 0.05,
+  "l_conv_atol": 0.0001,
+  "detach_every_n_steps": 32,
+  "h_halt_thresh": 0.9,
+  "encourage_thinking": false,
+  "adaptive_ponder": true,
+  "ponder_target_scale": 0.5,
+  "memory_token_routers": true,
+  "memory_gate_warmup_steps": 5000,
+  "memory_gate_warmup_floor": 0.1,
+  "reset_halt_bias": null,
+  "override_scheduling": false,
+  "persist_state": false,
+  "training_chunk_size": 256,
+  "cuda_loss_chunk_rows": 17136,
+  "cuda_chunked_lm_loss": true,
+  "cpu_loss_chunk_rows": 0,
+  "cpu_chunked_lm_loss": true,
+  "ltm_cpu_gather_retrieval": true,
+  "ltm_cpu_sparse_update": true,
+  "debug_numerics": false,
+  "save_steps": 600,
+  "progress_log_steps": 25,
+  "padding_metric_steps": 0,
+  "padding_metrics": true,
+  "num_workers": 8,
+  "prefetch_factor": null,
+  "pt_cache_size": 2,
+  "length_bucketing": true,
+  "length_bucket_size": 131072,
+  "auto_length_bucket_size": true,
+  "length_bucket_auto_tolerance": 0.005,
+  "streaming_datasets": true,
+  "hf_streaming_shuffle_buffer": 10000,
+  "hf_auto_shard": false,
+  "hf_shard_cache_dir": null,
+  "refresh_hf_shards": false,
+  "hf_cache_chunks_per_file": 2048,
+  "hf_token_cache": true,
+  "hf_token_cache_dir": null,
+  "refresh_hf_token_cache": false,
+  "amp": true,
+  "dataset_size": null,
+  "compile": true,
+  "force_compile": true,
+  "compile_mode": "max-autotune-no-cudagraphs",
+  "compile_dynamic": false,
+  "compile_backend": null,
+  "compile_fullgraph_worker": false,
+  "compile_cudagraphs": false,
+  "compile_pad_to_chunk_size": true,
+  "compile_static_worker_loop": null,
+  "compile_h_rnn": true,
+  "compile_quiet": true,
+  "eval_tasks": null,
+  "eval_every_epoch": 1,
+  "eval_batch_size": 1,
+  "eval_limit": null,
+  "eval_steps": null,
+  "benchmark_suite": null,
+  "benchmark": null,
+  "benchmark_all": false,
+  "benchmark_sequential": false,
+  "benchmark_out_dir": "./benchmark_results",
+  "benchmark_run_name": null,
+  "list_benchmarks": false,
+  "strict_benchmarks": false,
+  "arc_agi_path": null,
+  "arc_agi_max_tasks": null,
+  "arc_agi_max_test_items": null,
+  "arc_agi_keep_samples": false,
+  "temperature": 1.0,
+  "top_k": 40,
+  "top_p": 0.9,
+  "repetition_penalty": 1.2,
+  "max_new_tokens": 512,
+  "entropy_stop_threshold": 0.0,
+  "entropy_stop_min_tokens": 3,
+  "entropy_stop_top_prob": 0.05,
+  "eos_stop_prob": 0.0,
+  "device": null,
+  "threads": 24,
+  "enable_quantized_learning": false,
+  "ltm_lora_path": null,
+  "static_ltm_lr": true,
+  "ltm_schedule_steps": 100,
+  "ltm_schedule_min_lr": 1e-05,
+  "finetune_unlock_percent": null,
+  "gradient_checkpointing": false,
+  "passive_learning": true,
+  "passive_response_learning": false,
+  "passive_lr": 5e-06,
+  "surprise_threshold": 1.0,
+  "chat_state_file": null,
+  "resume_chat_from_state_file": null,
+  "chat_input_history_turns": 4,
+  "chat_input_history_chars": 3000,
+  "ckpt_input": null,
+  "inf_output": null,
+  "ckpt_tok_path": null,
+  "amp_dtype": "bfloat16",
+  "vocab_size": 50257,
+  "drift_norm_clamp": 4.0,
+  "drift_delta_scale": 0.35,
+  "rwkv_channel_mix_key_clamp": 12.0,
+  "ltm_training_mode": "read-only",
+  "rwkv_channel_mix_deepembed_clamp": 4.0,
+  "completed_epoch": 13
+}

tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 8880,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}