{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "<|im_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "<|im_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "<|object_ref_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 4, "content": "<|object_ref_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 5, "content": "<|box_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 6, "content": "<|box_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 7, "content": "<|quad_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 8, "content": "<|quad_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 9, "content": "<|vision_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 10, "content": "<|vision_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 11, "content": "<|vision_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 12, "content": "<|image_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 13, "content": "<|video_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 531, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 532, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 533, "content": "<|fim_prefix|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 534, "content": "<|fim_middle|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 535, "content": "<|fim_suffix|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 536, "content": "<|fim_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 537, "content": "<|repo_name|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 538, "content": "<|file_sep|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 539, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 540, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 541, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 542, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false } ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "post_processor": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": false, "use_regex": false }, "decoder": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "model": { "type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": "", "end_of_word_suffix": "", "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": { "<|endoftext|>": 0, "<|im_start|>": 1, "<|im_end|>": 2, "<|object_ref_start|>": 3, "<|object_ref_end|>": 4, "<|box_start|>": 5, "<|box_end|>": 6, "<|quad_start|>": 7, "<|quad_end|>": 8, "<|vision_start|>": 9, "<|vision_end|>": 10, "<|vision_pad|>": 11, "<|image_pad|>": 12, "<|video_pad|>": 13, "!": 14, "\"": 15, "#": 16, "$": 17, "%": 18, "&": 19, "'": 20, "(": 21, ")": 22, "*": 23, "+": 24, ",": 25, "-": 26, ".": 27, "/": 28, "0": 29, "1": 30, "2": 31, "3": 32, "4": 33, "5": 34, "6": 35, "7": 36, "8": 37, "9": 38, ":": 39, ";": 40, "<": 41, "=": 42, ">": 43, "?": 44, "@": 45, "A": 46, "B": 47, "C": 48, "D": 49, "E": 50, "F": 51, "G": 52, "H": 53, "I": 54, "J": 55, "K": 56, "L": 57, "M": 58, "N": 59, "O": 60, "P": 61, "Q": 62, "R": 63, "S": 64, "T": 65, "U": 66, "V": 67, "W": 68, "X": 69, "Y": 70, "Z": 71, "[": 72, "\\": 73, "]": 74, "^": 75, "_": 76, "`": 77, "a": 78, "b": 79, "c": 80, "d": 81, "e": 82, "f": 83, "g": 84, "h": 85, "i": 86, "j": 87, "k": 88, "l": 89, "m": 90, "n": 91, "o": 92, "p": 93, "q": 94, "r": 95, "s": 96, "t": 97, "u": 98, "v": 99, "w": 100, "x": 101, "y": 102, "z": 103, "{": 104, "|": 105, "}": 106, "~": 107, "¡": 108, "¢": 109, "£": 110, "¤": 111, "¥": 112, "¦": 113, "§": 114, "¨": 115, "©": 116, "ª": 117, "«": 118, "¬": 119, "®": 120, "¯": 121, "°": 122, "±": 123, "²": 124, "³": 125, "´": 126, "µ": 127, "¶": 128, "·": 129, "¸": 130, "¹": 131, "º": 132, "»": 133, "¼": 134, "½": 135, "¾": 136, "¿": 137, "À": 138, "Á": 139, "Â": 140, "Ã": 141, "Ä": 142, "Å": 143, "Æ": 144, "Ç": 145, "È": 146, "É": 147, "Ê": 148, "Ë": 149, "Ì": 150, "Í": 151, "Î": 152, "Ï": 153, "Ð": 154, "Ñ": 155, "Ò": 156, "Ó": 157, "Ô": 158, "Õ": 159, "Ö": 160, "×": 161, "Ø": 162, "Ù": 163, "Ú": 164, "Û": 165, "Ü": 166, "Ý": 167, "Þ": 168, "ß": 169, "à": 170, "á": 171, "â": 172, "ã": 173, "ä": 174, "å": 175, "æ": 176, "ç": 177, "è": 178, "é": 179, "ê": 180, "ë": 181, "ì": 182, "í": 183, "î": 184, "ï": 185, "ð": 186, "ñ": 187, "ò": 188, "ó": 189, "ô": 190, "õ": 191, "ö": 192, "÷": 193, "ø": 194, "ù": 195, "ú": 196, "û": 197, "ü": 198, "ý": 199, "þ": 200, "ÿ": 201, "Ā": 202, "ā": 203, "Ă": 204, "ă": 205, "Ą": 206, "ą": 207, "Ć": 208, "ć": 209, "Ĉ": 210, "ĉ": 211, "Ċ": 212, "ċ": 213, "Č": 214, "č": 215, "Ď": 216, "ď": 217, "Đ": 218, "đ": 219, "Ē": 220, "ē": 221, "Ĕ": 222, "ĕ": 223, "Ė": 224, "ė": 225, "Ę": 226, "ę": 227, "Ě": 228, "ě": 229, "Ĝ": 230, "ĝ": 231, "Ğ": 232, "ğ": 233, "Ġ": 234, "ġ": 235, "Ģ": 236, "ģ": 237, "Ĥ": 238, "ĥ": 239, "Ħ": 240, "ħ": 241, "Ĩ": 242, "ĩ": 243, "Ī": 244, "ī": 245, "Ĭ": 246, "ĭ": 247, "Į": 248, "į": 249, "İ": 250, "ı": 251, "IJ": 252, "ij": 253, "Ĵ": 254, "ĵ": 255, "Ķ": 256, "ķ": 257, "ĸ": 258, "Ĺ": 259, "ĺ": 260, "Ļ": 261, "ļ": 262, "Ľ": 263, "ľ": 264, "Ŀ": 265, "ŀ": 266, "Ł": 267, "ł": 268, "Ń": 269, "Ġa": 270, "in": 271, "ing": 272, "ou": 273, "th": 274, "ve": 275, "Ġc": 276, "Ġo": 277, "Ġs": 278, "an": 279, "re": 280, "Ġl": 281, "ea": 282, "gh": 283, "ri": 284, "Ġw": 285, "Ġth": 286, "ver": 287, "Ġthe": 288, "ds": 289, "igh": 290, "nd": 291, "on": 292, "or": 293, "se": 294, "st": 295, "ti": 296, "Ġf": 297, "Ġin": 298, "Ġri": 299, "Ġcl": 300, "Ġligh": 301, "ati": 302, "de": 303, "it": 304, "mn": 305, "mati": 306, "ra": 307, "ss": 308, "ting": 309, "ut": 310, "Ġb": 311, "Ġd": 312, "Ġm": 313, "Ġre": 314, "Ġab": 315, "Ġat": 316, "Ġand": 317, "oun": 318, "ouds": 319, "Ġon": 320, "Ġst": 321, "Ġfo": 322, "Ġriver": 323, "Ġclouds": 324, "matic": 325, "Ġred": 326, "!?": 327, "\"/": 328, "#$": 329, "%&": 330, "'\"/": 331, "()": 332, "*+": 333, ",:": 334, "-()": 335, ".,:": 336, ";!?": 337, "<>": 338, "=<>": 339, "@#$": 340, "AB": 341, "CD": 342, "EF": 343, "GH": 344, "IJ": 345, "KL": 346, "MN": 347, "OP": 348, "QR": 349, "ST": 350, "UV": 351, "WX": 352, "YZ": 353, "[]": 354, "\\@#$": 355, "^_": 356, "`^_": 357, "ap": 358, "aw": 359, "ain": 360, "aut": 361, "ce": 362, "cde": 363, "cap": 364, "dg": 365, "ding": 366, "dra": 367, "ep": 368, "er": 369, "et": 370, "ematic": 371, "ff": 372, "fu": 373, "fgh": 374, "gr": 375, "hor": 376, "ij": 377, "ith": 378, "iet": 379, "iff": 380, "kl": 381, "ld": 382, "man": 383, "no": 384, "ning": 385, "nse": 386, "op": 387, "ove": 388, "over": 389, "oon": 390, "oss": 391, "oman": 392, "por": 393, "qr": 394, "qu": 395, "ring": 396, "ron": 397, "ross": 398, "sh": 399, "tra": 400, "tain": 401, "ter": 402, "tning": 403, "uv": 404, "uti": 405, "umn": 406, "unse": 407, "wx": 408, "yz": 409, "youn": 410, "{}": 411, "|~": 412, "Ġ.,:": 413, "Ġgr": 414, "Ġhor": 415, "Ġpor": 416, "Ġqu": 417, "Ġyoun": 418, "Ġast": 419, "Ġaut": 420, "inematic": 421, "inter": 422, "ound": 423, "ouse": 424, "thouse": 425, "ves": 426, "Ġcover": 427, "Ġcross": 428, "Ġcinematic": 429, "Ġof": 430, "Ġover": 431, "Ġoce": 432, "Ġold": 433, "Ġsea": 434, "Ġsit": 435, "Ġsno": 436, "Ġsunse": 437, "ands": 438, "rest": 439, "ress": 440, "resh": 441, "Ġlea": 442, "Ġlands": 443, "earing": 444, "eauti": 445, "ridg": 446, "Ġwith": 447, "Ġwoman": 448, "Ġwinter": 449, "Ġwearing": 450, "one": 451, "orm": 452, "stuv": 453, "Ġfresh": 454, "Ġriding": 455, "Ġcliff": 456, "Ġlighting": 457, "Ġlightning": 458, "Ġlighthouse": 459, "deep": 460, "mnop": 461, "Ġbeauti": 462, "Ġbridg": 463, "Ġdaw": 464, "Ġdress": 465, "Ġmoun": 466, "Ġmoon": 467, "Ġabcde": 468, "Ġabove": 469, "Ġstone": 470, "Ġstorm": 471, "Ġfox": 472, "Ġforest": 473, "Ġrivers": 474, "%&*+": 475, "'\"/\\@#$": 476, "-()[]": 477, ";!?-()[]": 478, "=<>|~": 479, "ABCD": 480, "EFGH": 481, "IJKL": 482, "MNOP": 483, "QRST": 484, "UVWX": 485, "cape": 486, "dramatic": 487, "ful": 488, "fghij": 489, "klmnop": 490, "qrstuv": 491, "ronaut": 492, "trait": 493, "tains": 494, "wxyz": 495, "{}'\"/\\@#$": 496, "Ġ.,:;!?-()[]": 497, "Ġground": 498, "Ġhorse": 499, "Ġportrait": 500, "Ġquiet": 501, "Ġyoung": 502, "Ġastronaut": 503, "Ġautumn": 504, "Ġcovering": 505, "Ġcrossing": 506, "Ġocean": 507, "Ġsitting": 508, "Ġsnow": 509, "Ġsunset": 510, "Ġleaves": 511, "Ġlandscape": 512, "Ġbeautiful": 513, "Ġbridge": 514, "Ġdawn": 515, "Ġmountains": 516, "Ġabcdefghij": 517, "%&*+=<>|~": 518, "ABCDEFGH": 519, "IJKLMNOP": 520, "QRSTUVWX": 521, "klmnopqrstuv": 522, "{}'\"/\\@#$%&*+=<>|~": 523, "Ġ.,:;!?-()[]{}'\"/\\@#$%&*+=<>|~": 524, "Ġabcdefghijklmnopqrstuv": 525, "ABCDEFGHIJKLMNOP": 526, "QRSTUVWXYZ": 527, "Ġ.,:;!?-()[]{}'\"/\\@#$%&*+=<>|~`^_": 528, "Ġabcdefghijklmnopqrstuvwxyz": 529, "ABCDEFGHIJKLMNOPQRSTUVWXYZ": 530 }, "merges": [ [ "Ġ", "a" ], [ "i", "n" ], [ "in", "g" ], [ "o", "u" ], [ "t", "h" ], [ "v", "e" ], [ "Ġ", "c" ], [ "Ġ", "o" ], [ "Ġ", "s" ], [ "a", "n" ], [ "r", "e" ], [ "Ġ", "l" ], [ "e", "a" ], [ "g", "h" ], [ "r", "i" ], [ "Ġ", "w" ], [ "Ġ", "th" ], [ "ve", "r" ], [ "Ġth", "e" ], [ "d", "s" ], [ "i", "gh" ], [ "n", "d" ], [ "o", "n" ], [ "o", "r" ], [ "s", "e" ], [ "s", "t" ], [ "t", "i" ], [ "Ġ", "f" ], [ "Ġ", "in" ], [ "Ġ", "ri" ], [ "Ġc", "l" ], [ "Ġl", "igh" ], [ "a", "ti" ], [ "d", "e" ], [ "i", "t" ], [ "m", "n" ], [ "m", "ati" ], [ "r", "a" ], [ "s", "s" ], [ "t", "ing" ], [ "u", "t" ], [ "Ġ", "b" ], [ "Ġ", "d" ], [ "Ġ", "m" ], [ "Ġ", "re" ], [ "Ġa", "b" ], [ "Ġa", "t" ], [ "Ġa", "nd" ], [ "ou", "n" ], [ "ou", "ds" ], [ "Ġo", "n" ], [ "Ġs", "t" ], [ "Ġf", "o" ], [ "Ġri", "ver" ], [ "Ġcl", "ouds" ], [ "mati", "c" ], [ "Ġre", "d" ], [ "!", "?" ], [ "\"", "/" ], [ "#", "$" ], [ "%", "&" ], [ "'", "\"/" ], [ "(", ")" ], [ "*", "+" ], [ ",", ":" ], [ "-", "()" ], [ ".", ",:" ], [ ";", "!?" ], [ "<", ">" ], [ "=", "<>" ], [ "@", "#$" ], [ "A", "B" ], [ "C", "D" ], [ "E", "F" ], [ "G", "H" ], [ "I", "J" ], [ "K", "L" ], [ "M", "N" ], [ "O", "P" ], [ "Q", "R" ], [ "S", "T" ], [ "U", "V" ], [ "W", "X" ], [ "Y", "Z" ], [ "[", "]" ], [ "\\", "@#$" ], [ "^", "_" ], [ "`", "^_" ], [ "a", "p" ], [ "a", "w" ], [ "a", "in" ], [ "a", "ut" ], [ "c", "e" ], [ "c", "de" ], [ "c", "ap" ], [ "d", "g" ], [ "d", "ing" ], [ "d", "ra" ], [ "e", "p" ], [ "e", "r" ], [ "e", "t" ], [ "e", "matic" ], [ "f", "f" ], [ "f", "u" ], [ "f", "gh" ], [ "g", "r" ], [ "h", "or" ], [ "i", "j" ], [ "i", "th" ], [ "i", "et" ], [ "i", "ff" ], [ "k", "l" ], [ "l", "d" ], [ "m", "an" ], [ "n", "o" ], [ "n", "ing" ], [ "n", "se" ], [ "o", "p" ], [ "o", "ve" ], [ "o", "ver" ], [ "o", "on" ], [ "o", "ss" ], [ "o", "man" ], [ "p", "or" ], [ "q", "r" ], [ "q", "u" ], [ "r", "ing" ], [ "r", "on" ], [ "r", "oss" ], [ "s", "h" ], [ "t", "ra" ], [ "t", "ain" ], [ "t", "er" ], [ "t", "ning" ], [ "u", "v" ], [ "u", "ti" ], [ "u", "mn" ], [ "u", "nse" ], [ "w", "x" ], [ "y", "z" ], [ "y", "oun" ], [ "{", "}" ], [ "|", "~" ], [ "Ġ", ".,:" ], [ "Ġ", "gr" ], [ "Ġ", "hor" ], [ "Ġ", "por" ], [ "Ġ", "qu" ], [ "Ġ", "youn" ], [ "Ġa", "st" ], [ "Ġa", "ut" ], [ "in", "ematic" ], [ "in", "ter" ], [ "ou", "nd" ], [ "ou", "se" ], [ "th", "ouse" ], [ "ve", "s" ], [ "Ġc", "over" ], [ "Ġc", "ross" ], [ "Ġc", "inematic" ], [ "Ġo", "f" ], [ "Ġo", "ver" ], [ "Ġo", "ce" ], [ "Ġo", "ld" ], [ "Ġs", "ea" ], [ "Ġs", "it" ], [ "Ġs", "no" ], [ "Ġs", "unse" ], [ "an", "ds" ], [ "re", "st" ], [ "re", "ss" ], [ "re", "sh" ], [ "Ġl", "ea" ], [ "Ġl", "ands" ], [ "ea", "ring" ], [ "ea", "uti" ], [ "ri", "dg" ], [ "Ġw", "ith" ], [ "Ġw", "oman" ], [ "Ġw", "inter" ], [ "Ġw", "earing" ], [ "on", "e" ], [ "or", "m" ], [ "st", "uv" ], [ "Ġf", "resh" ], [ "Ġri", "ding" ], [ "Ġcl", "iff" ], [ "Ġligh", "ting" ], [ "Ġligh", "tning" ], [ "Ġligh", "thouse" ], [ "de", "ep" ], [ "mn", "op" ], [ "Ġb", "eauti" ], [ "Ġb", "ridg" ], [ "Ġd", "aw" ], [ "Ġd", "ress" ], [ "Ġm", "oun" ], [ "Ġm", "oon" ], [ "Ġab", "cde" ], [ "Ġab", "ove" ], [ "Ġst", "one" ], [ "Ġst", "orm" ], [ "Ġfo", "x" ], [ "Ġfo", "rest" ], [ "Ġriver", "s" ], [ "%&", "*+" ], [ "'\"/", "\\@#$" ], [ "-()", "[]" ], [ ";!?", "-()[]" ], [ "=<>", "|~" ], [ "AB", "CD" ], [ "EF", "GH" ], [ "IJ", "KL" ], [ "MN", "OP" ], [ "QR", "ST" ], [ "UV", "WX" ], [ "cap", "e" ], [ "dra", "matic" ], [ "fu", "l" ], [ "fgh", "ij" ], [ "kl", "mnop" ], [ "qr", "stuv" ], [ "ron", "aut" ], [ "tra", "it" ], [ "tain", "s" ], [ "wx", "yz" ], [ "{}", "'\"/\\@#$" ], [ "Ġ.,:", ";!?-()[]" ], [ "Ġgr", "ound" ], [ "Ġhor", "se" ], [ "Ġpor", "trait" ], [ "Ġqu", "iet" ], [ "Ġyoun", "g" ], [ "Ġast", "ronaut" ], [ "Ġaut", "umn" ], [ "Ġcover", "ing" ], [ "Ġcross", "ing" ], [ "Ġoce", "an" ], [ "Ġsit", "ting" ], [ "Ġsno", "w" ], [ "Ġsunse", "t" ], [ "Ġlea", "ves" ], [ "Ġlands", "cape" ], [ "Ġbeauti", "ful" ], [ "Ġbridg", "e" ], [ "Ġdaw", "n" ], [ "Ġmoun", "tains" ], [ "Ġabcde", "fghij" ], [ "%&*+", "=<>|~" ], [ "ABCD", "EFGH" ], [ "IJKL", "MNOP" ], [ "QRST", "UVWX" ], [ "klmnop", "qrstuv" ], [ "{}'\"/\\@#$", "%&*+=<>|~" ], [ "Ġ.,:;!?-()[]", "{}'\"/\\@#$%&*+=<>|~" ], [ "Ġabcdefghij", "klmnopqrstuv" ], [ "ABCDEFGH", "IJKLMNOP" ], [ "QRSTUVWX", "YZ" ], [ "Ġ.,:;!?-()[]{}'\"/\\@#$%&*+=<>|~", "`^_" ], [ "Ġabcdefghijklmnopqrstuv", "wxyz" ], [ "ABCDEFGHIJKLMNOP", "QRSTUVWXYZ" ] ] } }