v2 config: vocab 12288, 56,902,144 params

config.json

@@ -4,12 +4,12 @@
   "n_layers": 16,
   "n_heads": 8,
   "ffn_dim": 2048,
-  "vocab_size": 8192,
+  "vocab_size": 12288,
   "max_seq_len": 512,
   "tie_word_embeddings": true,
   "norm": "rmsnorm",
   "activation": "gelu",
   "attn": "causal",
   "dtype": "float32",
-  "n_params": 54804992
-}
+  "n_params": 56902144
+}