Upload tokenizer.json
Browse files- tokenizer.json +3 -0
tokenizer.json
CHANGED
|
@@ -52,6 +52,9 @@
|
|
| 52 |
"normalizer": {
|
| 53 |
"type": "Sequence",
|
| 54 |
"normalizers": [
|
|
|
|
|
|
|
|
|
|
| 55 |
{
|
| 56 |
"type": "BertNormalizer",
|
| 57 |
"clean_text": true,
|
|
|
|
| 52 |
"normalizer": {
|
| 53 |
"type": "Sequence",
|
| 54 |
"normalizers": [
|
| 55 |
+
{
|
| 56 |
+
"type": "NFD"
|
| 57 |
+
},
|
| 58 |
{
|
| 59 |
"type": "BertNormalizer",
|
| 60 |
"clean_text": true,
|