PermuFormer / tokenizer.json
ACDRepo's picture
Upload folder using huggingface_hub
eb03713 verified
Raw
History Blame Contribute Delete
5.55 kB
{
"version": "1.0",
"truncation": {
"direction": "Right",
"max_length": 1000,
"strategy": "LongestFirst",
"stride": 0
},
"padding": {
"strategy": "BatchLongest",
"direction": "Right",
"pad_to_multiple_of": null,
"pad_id": 185,
"pad_type_id": 0,
"pad_token": "<pad>"
},
"added_tokens": [
{
"id": 5,
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 184,
"content": "<unk>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 185,
"content": "<pad>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "Whitespace"
},
"post_processor": null,
"decoder": null,
"model": {
"type": "WordLevel",
"vocab": {
",": 0,
"-1": 1,
"1linebegin": 2,
"1lineend": 3,
"1linemake": 4,
"<|endoftext|>": 5,
"=": 6,
"Coxeterreducedexpressionbegin": 7,
"Coxeterreducedexpressionend": 8,
"Coxeterreducedexpressionmake": 9,
"False": 10,
"Inversionvectorbegin": 11,
"Inversionvectorend": 12,
"Inversionvectormake": 13,
"Lehmercodebegin": 14,
"Lehmercodeend": 15,
"Lehmercodemake": 16,
"RSKtableauxbegin": 17,
"RSKtableauxend": 18,
"RSKtableauxmake": 19,
"True": 20,
"[": 21,
"]": 22,
"avoidsbegin": 23,
"avoidsend": 24,
"avoidsmake": 25,
"commutatormake": 26,
"complementmake": 27,
"conjugatemake": 28,
"cyclenotationbegin": 29,
"cyclenotationend": 30,
"cyclenotationmake": 31,
"cycletypebegin": 32,
"cycletypeend": 33,
"cycletypemake": 34,
"descentsbegin": 35,
"descentsend": 36,
"descentsmake": 37,
"descenttestbegin": 38,
"descenttestend": 39,
"exponentbegin": 40,
"exponentend": 41,
"fixedpointsbegin": 42,
"fixedpointsend": 43,
"fixedpointsmake": 44,
"in": 45,
"indexbegin": 46,
"indexend": 47,
"inversemake": 48,
"isderangementbegin": 49,
"isderangementend": 50,
"isderangementmake": 51,
"isevenbegin": 52,
"isevenend": 53,
"isevenmake": 54,
"isgrassmannianbegin": 55,
"isgrassmannianend": 56,
"isgrassmannianmake": 57,
"isinvolutionbegin": 58,
"isinvolutionend": 59,
"isinvolutionmake": 60,
"ldslengthbegin": 61,
"ldslengthend": 62,
"ldslengthmake": 63,
"leftBruhatbegin": 64,
"leftBruhatend": 65,
"leftBruhatmake": 66,
"leftdescenttestmake": 67,
"lengthbegin": 68,
"lengthend": 69,
"lengthmake": 70,
"lislengthbegin": 71,
"lislengthend": 72,
"lislengthmake": 73,
"longestincreasingsubsequencebegin": 74,
"longestincreasingsubsequenceend": 75,
"longestincreasingsubsequencemake": 76,
"majorindexbegin": 77,
"majorindexend": 78,
"majorindexmake": 79,
"multiplyleftsimake": 80,
"multiplyrightsimake": 81,
"n10": 82,
"n11": 83,
"n3": 84,
"n4": 85,
"n5": 86,
"n6": 87,
"n7": 88,
"n8": 89,
"n9": 90,
"nochain": 91,
"nopattern": 92,
"numcyclesbegin": 93,
"numcyclesend": 94,
"numcyclesmake": 95,
"numdescentsbegin": 96,
"numdescentsend": 97,
"numdescentsmake": 98,
"numexcedancesbegin": 99,
"numexcedancesend": 100,
"numexcedancesmake": 101,
"numfixedpointsbegin": 102,
"numfixedpointsend": 103,
"numfixedpointsmake": 104,
"numinversionsbegin": 105,
"numinversionsend": 106,
"numinversionsmake": 107,
"orderbegin": 108,
"orderend": 109,
"ordermake": 110,
"peaksbegin": 111,
"peaksend": 112,
"peaksmake": 113,
"powermake": 114,
"property": 115,
"recoilsbegin": 116,
"recoilsend": 117,
"recoilsmake": 118,
"relativeleftmake": 119,
"relativerightmake": 120,
"reversemake": 121,
"rightdescenttestmake": 122,
"rskshapebegin": 123,
"rskshapeend": 124,
"rskshapemake": 125,
"signbegin": 126,
"signend": 127,
"signmake": 128,
"timesmake": 129,
"witnessbegin": 130,
"witnessend": 131,
"0": 132,
"1": 133,
"10": 134,
"11": 135,
"12": 136,
"13": 137,
"14": 138,
"15": 139,
"16": 140,
"17": 141,
"18": 142,
"19": 143,
"2": 144,
"20": 145,
"21": 146,
"22": 147,
"23": 148,
"24": 149,
"25": 150,
"26": 151,
"27": 152,
"28": 153,
"29": 154,
"3": 155,
"30": 156,
"31": 157,
"32": 158,
"33": 159,
"34": 160,
"35": 161,
"36": 162,
"37": 163,
"38": 164,
"39": 165,
"4": 166,
"40": 167,
"41": 168,
"42": 169,
"43": 170,
"44": 171,
"45": 172,
"46": 173,
"47": 174,
"48": 175,
"49": 176,
"5": 177,
"50": 178,
"51": 179,
"6": 180,
"7": 181,
"8": 182,
"9": 183,
"<unk>": 184,
"<pad>": 185
},
"unk_token": "<unk>"
}
}