| { | |
| "architectures": ["SwordiesGPT"], | |
| "model_type": "swordies-gpt", | |
| "vocab_size": 8192, | |
| "hidden_size": 448, | |
| "num_hidden_layers": 9, | |
| "num_attention_heads": 7, | |
| "intermediate_size": 1408, | |
| "max_position_embeddings": 512, | |
| "tie_word_embeddings": true, | |
| "norm": "rmsnorm", | |
| "activation": "gelu", | |
| "attn": "fused-qkv-causal", | |
| "num_parameters": 22487360, | |
| "dtype": "float32", | |
| "note": "Custom from-scratch GPT, not a transformers model. Load with load_model.py (custom loader). No SFT: single-stage pretraining only." | |
| } |