Upload folder using huggingface_hub

README.md

@@ -0,0 +1,61 @@
+---
+language:
+- code
+library_name: transformers
+pipeline_tag: text-generation
+tags:
+- html
+- css
+- javascript
+- code
+- llama
+- from-scratch
+---
+
+# WebCoder-100M
+
+A small decoder-only model specialized in HTML, CSS and JavaScript.
+
+## Architecture
+- Parameters: **99,894,528**
+- Layers: 11
+- Hidden size: 768
+- Attention heads: 12
+- Vocabulary: 28,672
+- Max context: 2,048
+- Training sequence length: 1,024
+
+## Training
+The model was initialized from scratch.
+
+1. Causal pre-training on HTML/CSS/JavaScript.
+2. Instruction fine-tuning on web-development instruction/code pairs.
+
+## Token accounting
+- Total processed: **582,209,140**
+- Pre-training: **568,246,272**
+- SFT processed: **13,962,868**
+- SFT supervised response tokens: **9,124,446**
+- Global cap: **2,000,000,000**
+
+## Data
+Pre-training: `bigcode/the-stack-smol-xl`, HTML/JavaScript/CSS subsets.
+
+Instruction tuning: `iamtarun/code_instructions_120k_alpaca`, filtered for web-development examples.
+
+Review upstream dataset cards and source licenses before commercial use.
+
+## Prompt format
+```text
+<|system|>
+You are WebCoder...<|end|>
+<|user|>
+Create a responsive landing page...<|end|>
+<|assistant|>
+...
+```
+
+## Limitations
+This is a roughly 100M-parameter model trained from scratch. Its quality depends
+strongly on how many tokens were actually processed. Generated code can contain
+bugs or security issues and should be reviewed.

config.json

@@ -0,0 +1,32 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 2048,
+  "max_position_embeddings": 2048,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 11,
+  "num_key_value_heads": 12,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.15.0",
+  "use_cache": true,
+  "vocab_size": 28672
+}

generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 1,
+  "do_sample": true,
+  "eos_token_id": 2,
+  "max_new_tokens": 768,
+  "pad_token_id": 0,
+  "repetition_penalty": 1.05,
+  "temperature": 0.7,
+  "top_k": 50,
+  "top_p": 0.92,
+  "transformers_version": "5.15.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a65d0f4dc02e2bfcf321b92feef28586196f625872c1773ded167512cf324b98
+size 399589328

tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|bos|>",
+  "eos_token": "<|eos|>",
+  "extra_special_tokens": [
+    "<|system|>",
+    "<|user|>",
+    "<|assistant|>",
+    "<|end|>"
+  ],
+  "model_max_length": 2048,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|unk|>"
+}

training_config.json

@@ -0,0 +1,39 @@
+{
+  "model_name": "WebCoder-100M",
+  "vocab_size": 28672,
+  "hidden_size": 768,
+  "intermediate_size": 2048,
+  "num_hidden_layers": 11,
+  "num_attention_heads": 12,
+  "num_key_value_heads": 12,
+  "max_position_embeddings": 2048,
+  "seq_len": 1024,
+  "docs_per_language": 10000,
+  "tokenizer_docs_per_language": 4000,
+  "max_chars_per_doc": 200000,
+  "micro_batch_size": 8,
+  "grad_accum_steps": 4,
+  "pretrain_lr": 0.0003,
+  "sft_lr": 8e-05,
+  "weight_decay": 0.1,
+  "grad_clip": 1.0,
+  "max_total_tokens": 2000000000,
+  "pretrain_token_cap": 1950000000,
+  "sft_token_cap": 50000000,
+  "pretrain_warmup_tokens": 2000000,
+  "pretrain_decay_tokens": 250000000,
+  "sft_warmup_tokens": 250000,
+  "sft_decay_tokens": 25000000,
+  "train_minutes": 55.0,
+  "reserve_sft_minutes": 10.0,
+  "checkpoint_every_updates": 500,
+  "log_every_updates": 20,
+  "sft_dataset": "iamtarun/code_instructions_120k_alpaca",
+  "sft_max_examples": 40000,
+  "sft_epochs": 2,
+  "work_dir": "/content/webcoder100m",
+  "use_google_drive": false,
+  "drive_dir": "/content/drive/MyDrive/WebCoder100M",
+  "hf_repo_name": "webcoder-100m-html-css-js",
+  "hf_private": false
+}

training_metrics.json

@@ -0,0 +1,12 @@
+{
+  "parameters": 99894528,
+  "vocab_size": 28672,
+  "context_window": 2048,
+  "training_sequence_length": 1024,
+  "total_tokens_processed": 582209140,
+  "pretrain_tokens_processed": 568246272,
+  "sft_tokens_processed": 13962868,
+  "sft_supervised_tokens": 9124446,
+  "global_token_cap": 2000000000,
+  "optimizer_updates": 19137
+}