Zihan428 commited on
Commit
ef85ce7
·
verified ·
1 Parent(s): e2d6902

Update multilingual tokenizer rupee placeholder

Browse files
grapheme_mtl_merged_expanded_v1.json CHANGED
@@ -1022,12 +1022,12 @@
1022
  },
1023
  {
1024
  "id": 2065,
1025
- "content": "[PLACEHOLDER45]",
1026
  "single_word": false,
1027
  "lstrip": false,
1028
  "rstrip": false,
1029
  "normalized": false,
1030
- "special": true
1031
  },
1032
  {
1033
  "id": 2107,
@@ -3145,7 +3145,7 @@
3145
  "[cj_8]": 2062,
3146
  "[cj_9]": 2063,
3147
  "[cj_.]": 2064,
3148
- "[PLACEHOLDER45]": 2065,
3149
  "‰": 2066,
3150
  "‱": 2067,
3151
  "′": 2068,
@@ -3803,4 +3803,4 @@
3803
  "ˌ ɐ"
3804
  ]
3805
  }
3806
- }
 
1022
  },
1023
  {
1024
  "id": 2065,
1025
+ "content": "",
1026
  "single_word": false,
1027
  "lstrip": false,
1028
  "rstrip": false,
1029
  "normalized": false,
1030
+ "special": false
1031
  },
1032
  {
1033
  "id": 2107,
 
3145
  "[cj_8]": 2062,
3146
  "[cj_9]": 2063,
3147
  "[cj_.]": 2064,
3148
+ "": 2065,
3149
  "‰": 2066,
3150
  "‱": 2067,
3151
  "′": 2068,
 
3803
  "ˌ ɐ"
3804
  ]
3805
  }
3806
+ }