hf-internal-testing
/

fuyu-tiny-tokenizer

Model card Files Files and versions

hf-transformers-bot commited on Jun 30

Commit

9e90e0b

·

verified ·

1 Parent(s): 6e83cc1

Update tokenizer_config.json

Files changed (1) hide show

tokenizer_config.json +2 -1

tokenizer_config.json CHANGED Viewed

@@ -8,7 +8,7 @@
       "single_word": false,
       "special": true
     },
-    "500": {
       "content": "|ENDOFTEXT|",
       "lstrip": false,
       "normalized": false,
@@ -27,6 +27,7 @@
   "legacy": true,
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": null,
   "processor_class": "FuyuProcessor",
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,

       "single_word": false,
       "special": true
     },
+    "508": {
       "content": "|ENDOFTEXT|",
       "lstrip": false,
       "normalized": false,
   "legacy": true,
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": null,
+  "padding_side": "right",
   "processor_class": "FuyuProcessor",
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,