| { |
| "config": { |
| "project": "tinyllm", |
| "hub": { |
| "user": "pythonstudentiam", |
| "model_repo_suffix": "tinyllm", |
| "ckpt_repo_suffix": "tinyllm-checkpoints", |
| "model_repo": "pythonstudentiam/tinyllm", |
| "ckpt_repo": "pythonstudentiam/tinyllm-checkpoints" |
| }, |
| "tokenizer": { |
| "vocab_size": 8192, |
| "model_type": "bpe", |
| "character_coverage": 1.0, |
| "train_sentences": 400000, |
| "max_sentence_length": 8192, |
| "unk_id": 0, |
| "bos_id": 1, |
| "eos_id": 2, |
| "pad_id": 3, |
| "unk_piece": "<unk>", |
| "bos_piece": "<s>", |
| "eos_piece": "</s>", |
| "pad_piece": "<pad>", |
| "im_start": "<|im_start|>", |
| "im_end": "<|im_end|>", |
| "user_defined_symbols": [ |
| "<|im_start|>", |
| "<|im_end|>" |
| ], |
| "chat_template": "{% for message in messages %}{{ '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}" |
| }, |
| "model": { |
| "hidden_size": 384, |
| "num_hidden_layers": 8, |
| "num_attention_heads": 6, |
| "num_key_value_heads": 2, |
| "intermediate_size": 1024, |
| "vocab_size": 8192, |
| "max_position_embeddings": 512, |
| "rope_theta": 10000.0, |
| "rms_norm_eps": 1e-05, |
| "tie_word_embeddings": true, |
| "attention_bias": false, |
| "mlp_bias": false, |
| "initializer_range": 0.02, |
| "head_dim": 64, |
| "kv_dim": 128, |
| "n_rep": 3, |
| "n_params": 15735168 |
| }, |
| "data": { |
| "dataset_id": "roneneldan/TinyStories", |
| "instruct_dataset_id": "roneneldan/TinyStoriesInstruct", |
| "train_split": "train", |
| "val_split": "validation", |
| "seq_len": 512, |
| "val_tokens": 1000000, |
| "shard_tokens": 25000000, |
| "seed": 1337 |
| }, |
| "train": { |
| "micro_batch_size": 32, |
| "grad_accum_steps": 4, |
| "max_steps": 2500, |
| "learning_rate": 0.0006, |
| "min_lr_ratio": 0.1, |
| "warmup_steps": 200, |
| "weight_decay": 0.1, |
| "beta1": 0.9, |
| "beta2": 0.95, |
| "grad_clip": 1.0, |
| "dtype": "fp16", |
| "compile_model": false, |
| "eval_every": 250, |
| "eval_batches": 40, |
| "sample_every": 500, |
| "log_every": 10, |
| "checkpoint_every": 500, |
| "keep_last_n_checkpoints": 2, |
| "seed": 1337, |
| "smoke_max_steps": 50, |
| "smoke_stories": 2000, |
| "tokens_per_step": 65536, |
| "total_tokens": 163840000, |
| "min_lr": 5.9999999999999995e-05 |
| }, |
| "sft": { |
| "micro_batch_size": 16, |
| "grad_accum_steps": 4, |
| "max_steps": 1500, |
| "learning_rate": 0.0001, |
| "min_lr_ratio": 0.1, |
| "warmup_steps": 50, |
| "weight_decay": 0.0, |
| "beta1": 0.9, |
| "beta2": 0.95, |
| "grad_clip": 1.0, |
| "seq_len": 512, |
| "ignore_index": -100, |
| "eval_every": 200, |
| "log_every": 10, |
| "checkpoint_every": 500, |
| "seed": 1337 |
| }, |
| "gen": { |
| "max_new_tokens": 256, |
| "temperature": 0.8, |
| "top_p": 0.95, |
| "top_k": 40, |
| "repetition_penalty": 1.1, |
| "eval_prompts": [ |
| "Once upon a time, there was a little girl named Lily.", |
| "Tom and Sara went to the park. They saw a big", |
| "The cat was very hungry, so it" |
| ], |
| "eval_instructions": [ |
| "Write a story about a lost puppy who finds its way home.", |
| "Write a short story using the words: ball, tree, happy.", |
| "Tell me a story about a brave little boat." |
| ] |
| }, |
| "quant": { |
| "levels": [ |
| "Q8_0", |
| "Q5_K_M", |
| "Q4_K_M" |
| ], |
| "perplexity_ctx": 512, |
| "perplexity_chunks": 40 |
| }, |
| "serve": { |
| "host": "127.0.0.1", |
| "port": 8080, |
| "threads": 4, |
| "ctx_size": 512, |
| "served_model_name": "tinyllm", |
| "default_quant": "Q8_0", |
| "llamacpp_build": "b10107", |
| "llamacpp_asset": "llama-b10107-bin-win-cpu-x64.zip", |
| "base_url": "http://127.0.0.1:8080/v1", |
| "llamacpp_url": "https://github.com/ggml-org/llama.cpp/releases/download/b10107/llama-b10107-bin-win-cpu-x64.zip" |
| }, |
| "derived": { |
| "head_dim": 64, |
| "kv_dim": 128, |
| "n_params": 15735168, |
| "param_breakdown": { |
| "embedding": 3145728, |
| "attention": 3145728, |
| "mlp": 9437184, |
| "layernorms": 6528, |
| "lm_head": 0, |
| "per_layer": 1573632, |
| "blocks_total": 12589056, |
| "non_embedding": 12589440, |
| "total": 15735168 |
| }, |
| "tokens_per_step": 65536, |
| "total_tokens": 163840000, |
| "flops_per_token": 94411008 |
| } |
| } |
| } |
