{
  "config": {
    "project": "tinyllm",
    "hub": {
      "user": "pythonstudentiam",
      "model_repo_suffix": "tinyllm",
      "ckpt_repo_suffix": "tinyllm-checkpoints",
      "model_repo": "pythonstudentiam/tinyllm",
      "ckpt_repo": "pythonstudentiam/tinyllm-checkpoints"
    },
    "tokenizer": {
      "vocab_size": 8192,
      "model_type": "bpe",
      "character_coverage": 1.0,
      "train_sentences": 400000,
      "max_sentence_length": 8192,
      "unk_id": 0,
      "bos_id": 1,
      "eos_id": 2,
      "pad_id": 3,
      "unk_piece": "<unk>",
      "bos_piece": "<s>",
      "eos_piece": "</s>",
      "pad_piece": "<pad>",
      "im_start": "<|im_start|>",
      "im_end": "<|im_end|>",
      "user_defined_symbols": [
        "<|im_start|>",
        "<|im_end|>"
      ],
      "chat_template": "{% for message in messages %}{{ '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}"
    },
    "model": {
      "hidden_size": 384,
      "num_hidden_layers": 8,
      "num_attention_heads": 6,
      "num_key_value_heads": 2,
      "intermediate_size": 1024,
      "vocab_size": 8192,
      "max_position_embeddings": 512,
      "rope_theta": 10000.0,
      "rms_norm_eps": 1e-05,
      "tie_word_embeddings": true,
      "attention_bias": false,
      "mlp_bias": false,
      "initializer_range": 0.02,
      "head_dim": 64,
      "kv_dim": 128,
      "n_rep": 3,
      "n_params": 15735168
    },
    "data": {
      "dataset_id": "roneneldan/TinyStories",
      "instruct_dataset_id": "roneneldan/TinyStoriesInstruct",
      "train_split": "train",
      "val_split": "validation",
      "seq_len": 512,
      "val_tokens": 1000000,
      "shard_tokens": 25000000,
      "seed": 1337
    },
    "train": {
      "micro_batch_size": 32,
      "grad_accum_steps": 4,
      "max_steps": 2500,
      "learning_rate": 0.0006,
      "min_lr_ratio": 0.1,
      "warmup_steps": 200,
      "weight_decay": 0.1,
      "beta1": 0.9,
      "beta2": 0.95,
      "grad_clip": 1.0,
      "dtype": "fp16",
      "compile_model": false,
      "eval_every": 250,
      "eval_batches": 40,
      "sample_every": 500,
      "log_every": 10,
      "checkpoint_every": 500,
      "keep_last_n_checkpoints": 2,
      "seed": 1337,
      "smoke_max_steps": 50,
      "smoke_stories": 2000,
      "tokens_per_step": 65536,
      "total_tokens": 163840000,
      "min_lr": 5.9999999999999995e-05
    },
    "sft": {
      "micro_batch_size": 16,
      "grad_accum_steps": 4,
      "max_steps": 1500,
      "learning_rate": 0.0001,
      "min_lr_ratio": 0.1,
      "warmup_steps": 50,
      "weight_decay": 0.0,
      "beta1": 0.9,
      "beta2": 0.95,
      "grad_clip": 1.0,
      "seq_len": 512,
      "ignore_index": -100,
      "eval_every": 200,
      "log_every": 10,
      "checkpoint_every": 500,
      "seed": 1337
    },
    "gen": {
      "max_new_tokens": 256,
      "temperature": 0.8,
      "top_p": 0.95,
      "top_k": 40,
      "repetition_penalty": 1.1,
      "eval_prompts": [
        "Once upon a time, there was a little girl named Lily.",
        "Tom and Sara went to the park. They saw a big",
        "The cat was very hungry, so it"
      ],
      "eval_instructions": [
        "Write a story about a lost puppy who finds its way home.",
        "Write a short story using the words: ball, tree, happy.",
        "Tell me a story about a brave little boat."
      ]
    },
    "quant": {
      "levels": [
        "Q8_0",
        "Q5_K_M",
        "Q4_K_M"
      ],
      "perplexity_ctx": 512,
      "perplexity_chunks": 40
    },
    "serve": {
      "host": "127.0.0.1",
      "port": 8080,
      "threads": 4,
      "ctx_size": 512,
      "served_model_name": "tinyllm",
      "default_quant": "Q8_0",
      "llamacpp_build": "b10107",
      "llamacpp_asset": "llama-b10107-bin-win-cpu-x64.zip",
      "base_url": "http://127.0.0.1:8080/v1",
      "llamacpp_url": "https://github.com/ggml-org/llama.cpp/releases/download/b10107/llama-b10107-bin-win-cpu-x64.zip"
    },
    "derived": {
      "head_dim": 64,
      "kv_dim": 128,
      "n_params": 15735168,
      "param_breakdown": {
        "embedding": 3145728,
        "attention": 3145728,
        "mlp": 9437184,
        "layernorms": 6528,
        "lm_head": 0,
        "per_layer": 1573632,
        "blocks_total": 12589056,
        "non_embedding": 12589440,
        "total": 15735168
      },
      "tokens_per_step": 65536,
      "total_tokens": 163840000,
      "flops_per_token": 94411008
    }
  }
}