Hugging Face's logo

Banaxi-Tech
/
BananaMind-2.1-MiniCoder

Text Generation
Safetensors
code
English
bananamind21_coder
causal-lm
custom-code
partially-looped
n-gram
custom_code
Model card Files
xet
 Community
1
Banaxi-Tech commited on
Commit
5fe3ede
·
verified ·
1 Parent(s): 330e6d4

Save 5% checkpoint at 1,500,512,256 tokens

Browse files
Files changed (6) hide show
  1. checkpoint_metadata.json +104 -0
  2. config.json +73 -0
  3. generation_config.json +7 -0
  4. model.safetensors +3 -0
  5. training_metrics.jsonl +0 -0
  6. training_state.pt +3 -0
checkpoint_metadata.json ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "variant": "minicoder",
3
+ "parameters": 24949999,
4
+ "transformer_parameters": 19950029,
5
+ "ngram_parameters": 4999970,
6
+ "architecture": {
7
+ "variant": "minicoder",
8
+ "vocab_size": 8192,
9
+ "hidden_size": 384,
10
+ "intermediate_size": 456,
11
+ "num_hidden_layers": 13,
12
+ "num_attention_heads": 12,
13
+ "num_key_value_heads": 1,
14
+ "head_dim": 32,
15
+ "ngram_buckets": 51699,
16
+ "ngram_rank_per_hash": 48,
17
+ "loop_schedule": [
18
+ 0,
19
+ 1,
20
+ 2,
21
+ 3,
22
+ 4,
23
+ 5,
24
+ 6,
25
+ 7,
26
+ 8,
27
+ 4,
28
+ 5,
29
+ 6,
30
+ 7,
31
+ 8,
32
+ 9,
33
+ 10,
34
+ 11,
35
+ 12
36
+ ],
37
+ "ngram_injection_schedule_positions": [
38
+ 4,
39
+ 9
40
+ ],
41
+ "refresh_kernel_size": 9,
42
+ "max_position_embeddings": 4096,
43
+ "rope_theta": 100000.0,
44
+ "rms_norm_eps": 1e-05,
45
+ "initializer_range": 0.02,
46
+ "use_cache": false,
47
+ "transformers_version": "5.16.1",
48
+ "architectures": null,
49
+ "output_hidden_states": false,
50
+ "return_dict": true,
51
+ "dtype": null,
52
+ "chunk_size_feed_forward": 0,
53
+ "is_encoder_decoder": false,
54
+ "id2label": {
55
+ "0": "LABEL_0",
56
+ "1": "LABEL_1"
57
+ },
58
+ "label2id": {
59
+ "LABEL_0": 0,
60
+ "LABEL_1": 1
61
+ },
62
+ "problem_type": null,
63
+ "_name_or_path": "",
64
+ "tie_word_embeddings": true,
65
+ "bos_token_id": 1,
66
+ "eos_token_id": 2,
67
+ "pad_token_id": 0,
68
+ "unk_token_id": 3,
69
+ "model_type": "bananamind21_coder",
70
+ "output_attentions": false
71
+ },
72
+ "training_percent": 5,
73
+ "step": 2862,
74
+ "total_steps": 57221,
75
+ "tokens_seen": 1500512256,
76
+ "target_tokens": 30000000000,
77
+ "tokens_per_full_step": 524288,
78
+ "final_step_supervised_tokens": 240640,
79
+ "world_size": 4,
80
+ "local_batch": 32,
81
+ "global_batch_sequences": 128,
82
+ "gpu_name": "NVIDIA H200",
83
+ "architecture_revision": "330e6d42645239a75d24104a69cd4283b9be3539",
84
+ "dataset_revisions": {
85
+ "stack_v3": "df4b205fbba4cc1c2fd1f205b10d66f730798bb9",
86
+ "fineweb_edu": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9"
87
+ },
88
+ "target_source_shares": {
89
+ "stack_v3": 0.75,
90
+ "fineweb_edu": 0.25
91
+ },
92
+ "target_source_tokens": {
93
+ "stack_v3": 22500000000,
94
+ "fineweb_edu": 7500000000
95
+ },
96
+ "actual_source_tokens": {
97
+ "stack_v3": 1125646336,
98
+ "fineweb_edu": 374865920
99
+ },
100
+ "muon_peak_lr": 0.02,
101
+ "adamw_peak_lr": 0.002,
102
+ "ngram_peak_lr": 0.001,
103
+ "elapsed_seconds_this_job": 1581.3037240505219
104
+ }
config.json ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "variant": "minicoder",
3
+ "vocab_size": 8192,
4
+ "hidden_size": 384,
5
+ "intermediate_size": 456,
6
+ "num_hidden_layers": 13,
7
+ "num_attention_heads": 12,
8
+ "num_key_value_heads": 1,
9
+ "head_dim": 32,
10
+ "ngram_buckets": 51699,
11
+ "ngram_rank_per_hash": 48,
12
+ "loop_schedule": [
13
+ 0,
14
+ 1,
15
+ 2,
16
+ 3,
17
+ 4,
18
+ 5,
19
+ 6,
20
+ 7,
21
+ 8,
22
+ 4,
23
+ 5,
24
+ 6,
25
+ 7,
26
+ 8,
27
+ 9,
28
+ 10,
29
+ 11,
30
+ 12
31
+ ],
32
+ "ngram_injection_schedule_positions": [
33
+ 4,
34
+ 9
35
+ ],
36
+ "refresh_kernel_size": 9,
37
+ "max_position_embeddings": 4096,
38
+ "rope_theta": 100000.0,
39
+ "rms_norm_eps": 1e-05,
40
+ "initializer_range": 0.02,
41
+ "use_cache": false,
42
+ "transformers_version": "5.16.1",
43
+ "architectures": [
44
+ "BananaMind21CoderForCausalLM"
45
+ ],
46
+ "output_hidden_states": false,
47
+ "return_dict": true,
48
+ "dtype": null,
49
+ "chunk_size_feed_forward": 0,
50
+ "is_encoder_decoder": false,
51
+ "id2label": {
52
+ "0": "LABEL_0",
53
+ "1": "LABEL_1"
54
+ },
55
+ "label2id": {
56
+ "LABEL_0": 0,
57
+ "LABEL_1": 1
58
+ },
59
+ "problem_type": null,
60
+ "_name_or_path": "Banaxi-Tech/BananaMind-2.1-MiniCoder",
61
+ "tie_word_embeddings": true,
62
+ "bos_token_id": 1,
63
+ "eos_token_id": 2,
64
+ "pad_token_id": 0,
65
+ "unk_token_id": 3,
66
+ "model_type": "bananamind21_coder",
67
+ "output_attentions": false,
68
+ "auto_map": {
69
+ "AutoConfig": "configuration_bananamind21_coder.BananaMind21CoderConfig",
70
+ "AutoModelForCausalLM": "modeling_bananamind21_coder.BananaMind21CoderForCausalLM"
71
+ },
72
+ "torch_dtype": "float32"
73
+ }
generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "pad_token_id": 0,
6
+ "transformers_version": "5"
7
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d2dc0d9e060d5ecb8f3696028e13c8b1ca511a626b17111a84460082ae82d0f2
3
+ size 113459196
training_metrics.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
training_state.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b5a4779947ed67dee4b4b761f21245b8ce31e18a92b60bc47f01d9070d30818d
3
+ size 246137190