add TILE as special token for image_token_ids compatibility

tokenizer.json

@@ -1,1010 +1,3 @@
-{
-  "version": "1.0",
-  "truncation": null,
-  "padding": {
-    "strategy": "BatchLongest",
-    "direction": "Left",
-    "pad_to_multiple_of": null,
-    "pad_id": 0,
-    "pad_type_id": 0,
-    "pad_token": "<PAD>"
-  },
-  "added_tokens": [
-    {
-      "id": 0,
-      "content": "<PAD>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 1,
-      "content": "<UNK>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 2,
-      "content": "<CLS>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 3,
-      "content": "<SEP>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 4,
-      "content": "<MASK_TOKEN>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 5,
-      "content": "<BOS_TOKEN>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 6,
-      "content": "<EOS_TOKEN>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 7,
-      "content": "<EOP_TOKEN>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 500,
-      "content": "<|START_OF_TURN_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 501,
-      "content": "<|END_OF_TURN_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 502,
-      "content": "<|YES_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 503,
-      "content": "<|NO_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 504,
-      "content": "<|GOOD_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 505,
-      "content": "<|BAD_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 506,
-      "content": "<|USER_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 507,
-      "content": "<|CHATBOT_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 508,
-      "content": "<|SYSTEM_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 509,
-      "content": "<|USER_0_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 510,
-      "content": "<|USER_1_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 511,
-      "content": "<|USER_2_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 512,
-      "content": "<|USER_3_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 513,
-      "content": "<|USER_4_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 514,
-      "content": "<|USER_5_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 515,
-      "content": "<|USER_6_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 516,
-      "content": "<|USER_7_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 517,
-      "content": "<|USER_8_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 518,
-      "content": "<|USER_9_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 519,
-      "content": "<|START_THINKING|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 520,
-      "content": "<|END_THINKING|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 521,
-      "content": "<|START_RESPONSE|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 522,
-      "content": "<|END_RESPONSE|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 523,
-      "content": "<|START_ACTION|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 524,
-      "content": "<|END_ACTION|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 525,
-      "content": "<|START_TOOL_RESULT|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 526,
-      "content": "<|END_TOOL_RESULT|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 527,
-      "content": "<|EXTRA_8_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 528,
-      "content": "<|NEW_FILE|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 529,
-      "content": "<|BEGINNING_OF_PREFIX_FIM_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 530,
-      "content": "<|BEGINNING_OF_MIDDLE_FIM_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 531,
-      "content": "<|BEGINNING_OF_SUFFIX_FIM_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 532,
-      "content": "<|END_OF_MIDDLE_FIM_TOKEN|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 533,
-      "content": "<|START_OF_IMG|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 534,
-      "content": "<|END_OF_IMG|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 535,
-      "content": "<|IMG_LINE_BREAK|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 536,
-      "content": "<|IMG_PATCH|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": false
-    }
-  ],
-  "normalizer": {
-    "type": "NFC"
-  },
-  "pre_tokenizer": {
-    "type": "Sequence",
-    "pretokenizers": [
-      {
-        "type": "Digits",
-        "individual_digits": true
-      },
-      {
-        "type": "ByteLevel",
-        "add_prefix_space": false,
-        "trim_offsets": true,
-        "use_regex": true
-      }
-    ]
-  },
-  "post_processor": {
-    "type": "TemplateProcessing",
-    "single": [
-      {
-        "SpecialToken": {
-          "id": "<BOS_TOKEN>",
-          "type_id": 0
-        }
-      },
-      {
-        "Sequence": {
-          "id": "A",
-          "type_id": 0
-        }
-      }
-    ],
-    "pair": [
-      {
-        "SpecialToken": {
-          "id": "<BOS_TOKEN>",
-          "type_id": 0
-        }
-      },
-      {
-        "Sequence": {
-          "id": "A",
-          "type_id": 0
-        }
-      },
-      {
-        "SpecialToken": {
-          "id": "<BOS_TOKEN>",
-          "type_id": 1
-        }
-      },
-      {
-        "Sequence": {
-          "id": "B",
-          "type_id": 1
-        }
-      }
-    ],
-    "special_tokens": {
-      "<BOS_TOKEN>": {
-        "id": "<BOS_TOKEN>",
-        "ids": [
-          5
-        ],
-        "tokens": [
-          "<BOS_TOKEN>"
-        ]
-      }
-    }
-  },
-  "decoder": {
-    "type": "ByteLevel",
-    "add_prefix_space": true,
-    "trim_offsets": true,
-    "use_regex": true
-  },
-  "model": {
-    "type": "BPE",
-    "dropout": null,
-    "unk_token": null,
-    "continuing_subword_prefix": null,
-    "end_of_word_suffix": null,
-    "fuse_unk": false,
-    "byte_fallback": false,
-    "ignore_merges": false,
-    "vocab": {
-      "<PAD>": 0,
-      "<UNK>": 1,
-      "<CLS>": 2,
-      "<SEP>": 3,
-      "<MASK_TOKEN>": 4,
-      "<BOS_TOKEN>": 5,
-      "<EOS_TOKEN>": 6,
-      "<EOP_TOKEN>": 7,
-      "!": 8,
-      "\"": 9,
-      "#": 10,
-      "$": 11,
-      "%": 12,
-      "&": 13,
-      "'": 14,
-      "(": 15,
-      ")": 16,
-      "*": 17,
-      "+": 18,
-      ",": 19,
-      "-": 20,
-      ".": 21,
-      "/": 22,
-      "0": 23,
-      "1": 24,
-      "2": 25,
-      "3": 26,
-      "4": 27,
-      "5": 28,
-      "6": 29,
-      "7": 30,
-      "8": 31,
-      "9": 32,
-      ":": 33,
-      ";": 34,
-      "<": 35,
-      "=": 36,
-      ">": 37,
-      "?": 38,
-      "@": 39,
-      "A": 40,
-      "B": 41,
-      "C": 42,
-      "D": 43,
-      "E": 44,
-      "F": 45,
-      "G": 46,
-      "H": 47,
-      "I": 48,
-      "J": 49,
-      "K": 50,
-      "L": 51,
-      "M": 52,
-      "N": 53,
-      "O": 54,
-      "P": 55,
-      "Q": 56,
-      "R": 57,
-      "S": 58,
-      "T": 59,
-      "U": 60,
-      "V": 61,
-      "W": 62,
-      "X": 63,
-      "Y": 64,
-      "Z": 65,
-      "[": 66,
-      "\\": 67,
-      "]": 68,
-      "^": 69,
-      "_": 70,
-      "`": 71,
-      "a": 72,
-      "b": 73,
-      "c": 74,
-      "d": 75,
-      "e": 76,
-      "f": 77,
-      "g": 78,
-      "h": 79,
-      "i": 80,
-      "j": 81,
-      "k": 82,
-      "l": 83,
-      "m": 84,
-      "n": 85,
-      "o": 86,
-      "p": 87,
-      "q": 88,
-      "r": 89,
-      "s": 90,
-      "t": 91,
-      "u": 92,
-      "v": 93,
-      "w": 94,
-      "x": 95,
-      "y": 96,
-      "z": 97,
-      "{": 98,
-      "|": 99,
-      "}": 100,
-      "~": 101,
-      "¡": 102,
-      "¢": 103,
-      "£": 104,
-      "¤": 105,
-      "¥": 106,
-      "¦": 107,
-      "§": 108,
-      "¨": 109,
-      "©": 110,
-      "ª": 111,
-      "«": 112,
-      "¬": 113,
-      "®": 114,
-      "¯": 115,
-      "°": 116,
-      "±": 117,
-      "²": 118,
-      "³": 119,
-      "´": 120,
-      "µ": 121,
-      "¶": 122,
-      "·": 123,
-      "¸": 124,
-      "¹": 125,
-      "º": 126,
-      "»": 127,
-      "¼": 128,
-      "½": 129,
-      "¾": 130,
-      "¿": 131,
-      "À": 132,
-      "Á": 133,
-      "Â": 134,
-      "Ã": 135,
-      "Ä": 136,
-      "Å": 137,
-      "Æ": 138,
-      "Ç": 139,
-      "È": 140,
-      "É": 141,
-      "Ê": 142,
-      "Ë": 143,
-      "Ì": 144,
-      "Í": 145,
-      "Î": 146,
-      "Ï": 147,
-      "Ð": 148,
-      "Ñ": 149,
-      "Ò": 150,
-      "Ó": 151,
-      "Ô": 152,
-      "Õ": 153,
-      "Ö": 154,
-      "×": 155,
-      "Ø": 156,
-      "Ù": 157,
-      "Ú": 158,
-      "Û": 159,
-      "Ü": 160,
-      "Ý": 161,
-      "Þ": 162,
-      "ß": 163,
-      "à": 164,
-      "á": 165,
-      "â": 166,
-      "ã": 167,
-      "ä": 168,
-      "å": 169,
-      "æ": 170,
-      "ç": 171,
-      "è": 172,
-      "é": 173,
-      "ê": 174,
-      "ë": 175,
-      "ì": 176,
-      "í": 177,
-      "î": 178,
-      "ï": 179,
-      "ð": 180,
-      "ñ": 181,
-      "ò": 182,
-      "ó": 183,
-      "ô": 184,
-      "õ": 185,
-      "ö": 186,
-      "÷": 187,
-      "ø": 188,
-      "ù": 189,
-      "ú": 190,
-      "û": 191,
-      "ü": 192,
-      "ý": 193,
-      "þ": 194,
-      "ÿ": 195,
-      "Ā": 196,
-      "ā": 197,
-      "Ă": 198,
-      "ă": 199,
-      "Ą": 200,
-      "ą": 201,
-      "Ć": 202,
-      "ć": 203,
-      "Ĉ": 204,
-      "ĉ": 205,
-      "Ċ": 206,
-      "ċ": 207,
-      "Č": 208,
-      "č": 209,
-      "Ď": 210,
-      "ď": 211,
-      "Đ": 212,
-      "đ": 213,
-      "Ē": 214,
-      "ē": 215,
-      "Ĕ": 216,
-      "ĕ": 217,
-      "Ė": 218,
-      "ė": 219,
-      "Ę": 220,
-      "ę": 221,
-      "Ě": 222,
-      "ě": 223,
-      "Ĝ": 224,
-      "ĝ": 225,
-      "Ğ": 226,
-      "ğ": 227,
-      "Ġ": 228,
-      "ġ": 229,
-      "Ģ": 230,
-      "ģ": 231,
-      "Ĥ": 232,
-      "ĥ": 233,
-      "Ħ": 234,
-      "ħ": 235,
-      "Ĩ": 236,
-      "ĩ": 237,
-      "Ī": 238,
-      "ī": 239,
-      "Ĭ": 240,
-      "ĭ": 241,
-      "Į": 242,
-      "į": 243,
-      "İ": 244,
-      "ı": 245,
-      "IJ": 246,
-      "ij": 247,
-      "Ĵ": 248,
-      "ĵ": 249,
-      "Ķ": 250,
-      "ķ": 251,
-      "ĸ": 252,
-      "Ĺ": 253,
-      "ĺ": 254,
-      "Ļ": 255,
-      "ļ": 256,
-      "Ľ": 257,
-      "ľ": 258,
-      "Ŀ": 259,
-      "ŀ": 260,
-      "Ł": 261,
-      "ł": 262,
-      "Ń": 263,
-      "‍": 264,
-      "‼": 265,
-      "⁉": 266,
-      "⃣": 267,
-      "™": 268,
-      "ℹ": 269,
-      "↔": 270,
-      "↕": 271,
-      "↖": 272,
-      "↗": 273,
-      "↘": 274,
-      "↙": 275,
-      "↩": 276,
-      "↪": 277,
-      "⌚": 278,
-      "⌛": 279,
-      "⌨": 280,
-      "⏏": 281,
-      "⏩": 282,
-      "⏪": 283,
-      "⏫": 284,
-      "⏬": 285,
-      "⏭": 286,
-      "⏮": 287,
-      "⏯": 288,
-      "⏰": 289,
-      "⏱": 290,
-      "⏲": 291,
-      "⏳": 292,
-      "⏸": 293,
-      "⏹": 294,
-      "⏺": 295,
-      "Ⓜ": 296,
-      "▪": 297,
-      "▫": 298,
-      "▶": 299,
-      "◀": 300,
-      "◻": 301,
-      "◼": 302,
-      "◽": 303,
-      "◾": 304,
-      "☀": 305,
-      "☁": 306,
-      "☂": 307,
-      "☃": 308,
-      "☄": 309,
-      "☎": 310,
-      "☑": 311,
-      "☔": 312,
-      "☕": 313,
-      "☘": 314,
-      "☝": 315,
-      "☠": 316,
-      "☢": 317,
-      "☣": 318,
-      "☦": 319,
-      "☪": 320,
-      "☮": 321,
-      "☯": 322,
-      "☸": 323,
-      "☹": 324,
-      "☺": 325,
-      "♀": 326,
-      "♂": 327,
-      "♈": 328,
-      "♉": 329,
-      "♊": 330,
-      "♋": 331,
-      "♌": 332,
-      "♍": 333,
-      "♎": 334,
-      "♏": 335,
-      "♐": 336,
-      "♑": 337,
-      "♒": 338,
-      "♓": 339,
-      "♟": 340,
-      "♠": 341,
-      "♣": 342,
-      "♥": 343,
-      "♦": 344,
-      "♨": 345,
-      "♻": 346,
-      "♾": 347,
-      "♿": 348,
-      "⚒": 349,
-      "⚓": 350,
-      "⚔": 351,
-      "⚕": 352,
-      "⚖": 353,
-      "⚗": 354,
-      "⚙": 355,
-      "⚛": 356,
-      "⚜": 357,
-      "⚠": 358,
-      "⚡": 359,
-      "⚧": 360,
-      "⚪": 361,
-      "⚫": 362,
-      "⚰": 363,
-      "⚱": 364,
-      "⚽": 365,
-      "⚾": 366,
-      "⛄": 367,
-      "⛅": 368,
-      "⛈": 369,
-      "⛎": 370,
-      "⛏": 371,
-      "⛑": 372,
-      "⛓": 373,
-      "⛔": 374,
-      "⛩": 375,
-      "⛪": 376,
-      "⛰": 377,
-      "⛱": 378,
-      "⛲": 379,
-      "⛳": 380,
-      "⛴": 381,
-      "⛵": 382,
-      "⛷": 383,
-      "⛸": 384,
-      "⛹": 385,
-      "⛺": 386,
-      "⛽": 387,
-      "✂": 388,
-      "✅": 389,
-      "✈": 390,
-      "✉": 391,
-      "✊": 392,
-      "✋": 393,
-      "✌": 394,
-      "✍": 395,
-      "✏": 396,
-      "✒": 397,
-      "✔": 398,
-      "✖": 399,
-      "✝": 400,
-      "✡": 401,
-      "✨": 402,
-      "✳": 403,
-      "✴": 404,
-      "❄": 405,
-      "❇": 406,
-      "❌": 407,
-      "❎": 408,
-      "❓": 409,
-      "❔": 410,
-      "❕": 411,
-      "❗": 412,
-      "❣": 413,
-      "❤": 414,
-      "➕": 415,
-      "➖": 416,
-      "➗": 417,
-      "➡": 418,
-      "➰": 419,
-      "➿": 420,
-      "⤴": 421,
-      "⤵": 422,
-      "⬅": 423,
-      "⬆": 424,
-      "⬇": 425,
-      "⬛": 426,
-      "⬜": 427,
-      "⭐": 428,
-      "⭕": 429,
-      "〰": 430,
-      "〽": 431,
-      "㊗": 432,
-      "㊙": 433,
-      "️": 434,
-      "🀄": 435,
-      "🃏": 436,
-      "🅰": 437,
-      "🅱": 438,
-      "🅾": 439,
-      "🅿": 440,
-      "🆎": 441,
-      "🆑": 442,
-      "🆒": 443,
-      "🆓": 444,
-      "🆔": 445,
-      "🆕": 446,
-      "🆖": 447,
-      "🆗": 448,
-      "🆘": 449,
-      "🆙": 450,
-      "🆚": 451,
-      "🇦": 452,
-      "🇧": 453,
-      "🇨": 454,
-      "🇩": 455,
-      "🇪": 456,
-      "🇫": 457,
-      "🇬": 458,
-      "🇭": 459,
-      "🇮": 460,
-      "🇯": 461,
-      "🇰": 462,
-      "🇱": 463,
-      "🇲": 464,
-      "🇳": 465,
-      "🇴": 466,
-      "🇵": 467,
-      "🇶": 468,
-      "🇷": 469,
-      "🇸": 470,
-      "🇹": 471,
-      "🇺": 472,
-      "🇻": 473,
-      "🇼": 474,
-      "🇽": 475,
-      "🇾": 476,
-      "🇿": 477,
-      "🈁": 478,
-      "🈂": 479,
-      "🈚": 480,
-      "🈯": 481,
-      "🈲": 482,
-      "🈳": 483,
-      "🈴": 484,
-      "🈵": 485,
-      "🈶": 486,
-      "🈷": 487,
-      "🈸": 488,
-      "🈹": 489,
-      "🈺": 490,
-      "🉐": 491,
-      "🉑": 492,
-      "🌀": 493,
-      "🌁": 494,
-      "🌂": 495,
-      "🌃": 496,
-      "🌄": 497,
-      "🌅": 498,
-      "🌆": 499
-    },
-    "merges": []
-  }
-}
+version https://git-lfs.github.com/spec/v1
+oid sha256:d43a944b2080b27445b7109220cf008f39b58852ebaede63735727b226ef9ee9
+size 19244