data-ladder tokenized: c4_neox/meta.json
Browse files
data-ladder/c4_neox/meta.json
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "c4",
|
| 3 |
+
"tokenizer": "EleutherAI/gpt-neox-20b",
|
| 4 |
+
"tiktoken_encoding": "EleutherAI/gpt-neox-20b",
|
| 5 |
+
"eot_id": 0,
|
| 6 |
+
"dtype": "uint16",
|
| 7 |
+
"seed": 1234,
|
| 8 |
+
"shuffle_buffer": 0,
|
| 9 |
+
"val_tokens": 5000356,
|
| 10 |
+
"val_bytes": 22675974,
|
| 11 |
+
"val_docs": 10510,
|
| 12 |
+
"train_tokens": 9100000116,
|
| 13 |
+
"train_bytes": 41206328548,
|
| 14 |
+
"train_docs": 19007958
|
| 15 |
+
}
|