Upload tokenizer
Browse files- tokenizer.json +10 -1
tokenizer.json
CHANGED
|
@@ -2879,7 +2879,16 @@
|
|
| 2879 |
"<p2698>": 2836,
|
| 2880 |
"<p2699>": 2837,
|
| 2881 |
"<p2700>": 2838,
|
| 2882 |
-
"<UNK>": 2839
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2883 |
},
|
| 2884 |
"unk_token": "<UNK>"
|
| 2885 |
}
|
|
|
|
| 2879 |
"<p2698>": 2836,
|
| 2880 |
"<p2699>": 2837,
|
| 2881 |
"<p2700>": 2838,
|
| 2882 |
+
"<UNK>": 2839,
|
| 2883 |
+
"<deterministic-positives-only>": 2840,
|
| 2884 |
+
"<random-3-bins>": 2841,
|
| 2885 |
+
"<begin_contacts>": 2842,
|
| 2886 |
+
"<end_contacts>": 2843,
|
| 2887 |
+
"<bin_lt4>": 2844,
|
| 2888 |
+
"<bin_4_12>": 2845,
|
| 2889 |
+
"<bin_gt12>": 2846,
|
| 2890 |
+
"<correction>": 2847,
|
| 2891 |
+
"<non-correction>": 2848
|
| 2892 |
},
|
| 2893 |
"unk_token": "<UNK>"
|
| 2894 |
}
|