Hugging Face's logo

aklein4
/
Horizon-TPU_batch-test

Model card Files
xet
 Community
aklein4 commited on
Commit
ba28595
·
verified ·
1 Parent(s): 4e3e5cc

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. 000000000400/config.json +132 -0
  2. 000000000400/model.pt +3 -0
000000000400/config.json ADDED
@@ -0,0 +1,132 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "type": "forte.ForteModel",
3
+ "pretrained_url": "aklein4/forte-init",
4
+ "pretrained_step": 0,
5
+ "pretrained_strict": true,
6
+ "torch_dtype": "float32",
7
+ "vocab_size": 128256,
8
+ "bos_token_id": 128000,
9
+ "eos_token_id": 128001,
10
+ "pad_token_id": -100,
11
+ "hidden_size": 2048,
12
+ "num_hidden_layers": 16,
13
+ "num_attention_heads": 32,
14
+ "num_key_value_heads": 8,
15
+ "intermediate_size": 8192,
16
+ "hidden_act": "silu",
17
+ "max_position_embeddings": 131072,
18
+ "rope_theta": 500000.0,
19
+ "rope_scaling": {
20
+ "factor": 32.0,
21
+ "high_freq_factor": 4.0,
22
+ "low_freq_factor": 1.0,
23
+ "original_context_len": 8192
24
+ },
25
+ "initializer_range": 0.02,
26
+ "attention_dropout": false,
27
+ "attention_bias": false,
28
+ "rms_norm_eps": 1e-05,
29
+ "attention_kernel": "flash_attention",
30
+ "pure_modules": [],
31
+ "sharding": {
32
+ "replicate_default": true,
33
+ "output_layers.layers.*.mlp.gate_proj.weight": [
34
+ "fsdp",
35
+ null
36
+ ],
37
+ "output_layers.layers.*.mlp.up_proj.weight": [
38
+ "fsdp",
39
+ null
40
+ ],
41
+ "output_layers.layers.*.mlp.down_proj.weight": [
42
+ "fsdp",
43
+ null
44
+ ],
45
+ "bidirectional_head.layers.layers.*.mlp.gate_proj.weight": [
46
+ "fsdp",
47
+ null
48
+ ],
49
+ "bidirectional_head.layers.layers.*.mlp.up_proj.weight": [
50
+ "fsdp",
51
+ null
52
+ ],
53
+ "bidirectional_head.layers.layers.*.mlp.down_proj.weight": [
54
+ "fsdp",
55
+ null
56
+ ],
57
+ "backbone_layers.layers.*": [
58
+ [
59
+ "data",
60
+ "fsdp"
61
+ ],
62
+ null,
63
+ null
64
+ ],
65
+ "output_layers.layers.*": [
66
+ [
67
+ "data",
68
+ "fsdp"
69
+ ],
70
+ null,
71
+ null
72
+ ],
73
+ "bidirectional_head.layers.layers.*": [
74
+ [
75
+ "data",
76
+ "fsdp"
77
+ ],
78
+ null,
79
+ null
80
+ ]
81
+ },
82
+ "remat": {
83
+ "advanced": [
84
+ {
85
+ "name": "backbone_layers",
86
+ "settings": {
87
+ "activation_checkpoint_layers": [
88
+ "ForteBackboneLayer"
89
+ ],
90
+ "optimization_barrier_layers": [
91
+ "ForteBackboneLayer"
92
+ ],
93
+ "scan_layers": "layers",
94
+ "is_layer_pure": true
95
+ }
96
+ },
97
+ {
98
+ "name": "output_layers",
99
+ "settings": {
100
+ "activation_checkpoint_layers": [
101
+ "ForteOutputLayer"
102
+ ],
103
+ "optimization_barrier_layers": [
104
+ "ForteOutputLayer"
105
+ ],
106
+ "scan_layers": "layers",
107
+ "is_layer_pure": true
108
+ }
109
+ },
110
+ {
111
+ "name": "bidirectional_head",
112
+ "settings": {
113
+ "activation_checkpoint_layers": [
114
+ "BidirectionalDecoderLayer"
115
+ ],
116
+ "optimization_barrier_layers": [
117
+ "BidirectionalDecoderLayer"
118
+ ],
119
+ "scan_layers": "layers.layers",
120
+ "is_layer_pure": true
121
+ }
122
+ }
123
+ ]
124
+ },
125
+ "fast_weight_size": 1536,
126
+ "base_lr": 0.001,
127
+ "grad_rms_eps": 1e-12,
128
+ "num_output_layers": 4,
129
+ "num_bidirectional_layers": 6,
130
+ "mixer_kernel_size": 5,
131
+ "pad_attention_bias_value": -1000.0
132
+ }
000000000400/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:397e064abea8b0cecaa3c55d1b770a8bf8b0cff782e484cbe771df413af6cb30
3
+ size 8238143199