Fix tokenizer for transformers 4.x: add vocab.json/merges.txt, drop 5.x-only config keys (#2)

- Fix tokenizer for transformers 4.x: add vocab.json/merges.txt, drop 5.x-only config keys (5f619576a9c91dd0a30a7568623f06f02abd32f2)

tokenizer/merges.txt

@@ -0,0 +1,262 @@
+#version: 0.2
+Ġ a
+i n
+in g
+o u
+t h
+v e
+Ġ c
+Ġ o
+Ġ s
+a n
+r e
+Ġ l
+e a
+g h
+r i
+Ġ w
+Ġ th
+ve r
+Ġth e
+d s
+i gh
+n d
+o n
+o r
+s e
+s t
+t i
+Ġ f
+Ġ in
+Ġ ri
+Ġc l
+Ġl igh
+a ti
+d e
+i t
+m n
+m ati
+r a
+s s
+t ing
+u t
+Ġ b
+Ġ d
+Ġ m
+Ġ re
+Ġa b
+Ġa t
+Ġa nd
+ou n
+ou ds
+Ġo n
+Ġs t
+Ġf o
+Ġri ver
+Ġcl ouds
+mati c
+Ġre d
+! ?
+" /
+# $
+% &
+' "/
+( )
+* +
+, :
+- ()
+. ,:
+; !?
+< >
+= <>
+@ #$
+A B
+C D
+E F
+G H
+I J
+K L
+M N
+O P
+Q R
+S T
+U V
+W X
+Y Z
+[ ]
+\ @#$
+^ _
+` ^_
+a p
+a w
+a in
+a ut
+c e
+c de
+c ap
+d g
+d ing
+d ra
+e p
+e r
+e t
+e matic
+f f
+f u
+f gh
+g r
+h or
+i j
+i th
+i et
+i ff
+k l
+l d
+m an
+n o
+n ing
+n se
+o p
+o ve
+o ver
+o on
+o ss
+o man
+p or
+q r
+q u
+r ing
+r on
+r oss
+s h
+t ra
+t ain
+t er
+t ning
+u v
+u ti
+u mn
+u nse
+w x
+y z
+y oun
+{ }
+| ~
+Ġ .,:
+Ġ gr
+Ġ hor
+Ġ por
+Ġ qu
+Ġ youn
+Ġa st
+Ġa ut
+in ematic
+in ter
+ou nd
+ou se
+th ouse
+ve s
+Ġc over
+Ġc ross
+Ġc inematic
+Ġo f
+Ġo ver
+Ġo ce
+Ġo ld
+Ġs ea
+Ġs it
+Ġs no
+Ġs unse
+an ds
+re st
+re ss
+re sh
+Ġl ea
+Ġl ands
+ea ring
+ea uti
+ri dg
+Ġw ith
+Ġw oman
+Ġw inter
+Ġw earing
+on e
+or m
+st uv
+Ġf resh
+Ġri ding
+Ġcl iff
+Ġligh ting
+Ġligh tning
+Ġligh thouse
+de ep
+mn op
+Ġb eauti
+Ġb ridg
+Ġd aw
+Ġd ress
+Ġm oun
+Ġm oon
+Ġab cde
+Ġab ove
+Ġst one
+Ġst orm
+Ġfo x
+Ġfo rest
+Ġriver s
+%& *+
+'"/ \@#$
+-() []
+;!? -()[]
+=<> |~
+AB CD
+EF GH
+IJ KL
+MN OP
+QR ST
+UV WX
+cap e
+dra matic
+fu l
+fgh ij
+kl mnop
+qr stuv
+ron aut
+tra it
+tain s
+wx yz
+{} '"/\@#$
+Ġ.,: ;!?-()[]
+Ġgr ound
+Ġhor se
+Ġpor trait
+Ġqu iet
+Ġyoun g
+Ġast ronaut
+Ġaut umn
+Ġcover ing
+Ġcross ing
+Ġoce an
+Ġsit ting
+Ġsno w
+Ġsunse t
+Ġlea ves
+Ġlands cape
+Ġbeauti ful
+Ġbridg e
+Ġdaw n
+Ġmoun tains
+Ġabcde fghij
+%&*+ =<>|~
+ABCD EFGH
+IJKL MNOP
+QRST UVWX
+klmnop qrstuv
+{}'"/\@#$ %&*+=<>|~
+Ġ.,:;!?-()[] {}'"/\@#$%&*+=<>|~
+Ġabcdefghij klmnopqrstuv
+ABCDEFGH IJKLMNOP
+QRSTUVWX YZ
+Ġ.,:;!?-()[]{}'"/\@#$%&*+=<>|~ `^_
+Ġabcdefghijklmnopqrstuv wxyz
+ABCDEFGHIJKLMNOP QRSTUVWXYZ

tokenizer/tokenizer_config.json

@@ -1,26 +1,9 @@
 {
   "add_prefix_space": false,
-  "backend": "tokenizers",
   "bos_token": null,
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
   "errors": "replace",
-  "extra_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>",
-    "<|object_ref_start|>",
-    "<|object_ref_end|>",
-    "<|box_start|>",
-    "<|box_end|>",
-    "<|quad_start|>",
-    "<|quad_end|>",
-    "<|vision_start|>",
-    "<|vision_end|>",
-    "<|vision_pad|>",
-    "<|image_pad|>",
-    "<|video_pad|>"
-  ],
-  "is_local": false,
   "model_max_length": 131072,
   "pad_token": "<|endoftext|>",
   "split_special_tokens": false,

tokenizer/vocab.json

@@ -0,0 +1 @@
+{"<|endoftext|>":0,"<|im_start|>":1,"<|im_end|>":2,"<|object_ref_start|>":3,"<|object_ref_end|>":4,"<|box_start|>":5,"<|box_end|>":6,"<|quad_start|>":7,"<|quad_end|>":8,"<|vision_start|>":9,"<|vision_end|>":10,"<|vision_pad|>":11,"<|image_pad|>":12,"<|video_pad|>":13,"!":14,"\"":15,"#":16,"$":17,"%":18,"&":19,"'":20,"(":21,")":22,"*":23,"+":24,",":25,"-":26,".":27,"/":28,"0":29,"1":30,"2":31,"3":32,"4":33,"5":34,"6":35,"7":36,"8":37,"9":38,":":39,";":40,"<":41,"=":42,">":43,"?":44,"@":45,"A":46,"B":47,"C":48,"D":49,"E":50,"F":51,"G":52,"H":53,"I":54,"J":55,"K":56,"L":57,"M":58,"N":59,"O":60,"P":61,"Q":62,"R":63,"S":64,"T":65,"U":66,"V":67,"W":68,"X":69,"Y":70,"Z":71,"[":72,"\\":73,"]":74,"^":75,"_":76,"`":77,"a":78,"b":79,"c":80,"d":81,"e":82,"f":83,"g":84,"h":85,"i":86,"j":87,"k":88,"l":89,"m":90,"n":91,"o":92,"p":93,"q":94,"r":95,"s":96,"t":97,"u":98,"v":99,"w":100,"x":101,"y":102,"z":103,"{":104,"|":105,"}":106,"~":107,"¡":108,"¢":109,"£":110,"¤":111,"¥":112,"¦":113,"§":114,"¨":115,"©":116,"ª":117,"«":118,"¬":119,"®":120,"¯":121,"°":122,"±":123,"²":124,"³":125,"´":126,"µ":127,"¶":128,"·":129,"¸":130,"¹":131,"º":132,"»":133,"¼":134,"½":135,"¾":136,"¿":137,"À":138,"Á":139,"Â":140,"Ã":141,"Ä":142,"Å":143,"Æ":144,"Ç":145,"È":146,"É":147,"Ê":148,"Ë":149,"Ì":150,"Í":151,"Î":152,"Ï":153,"Ð":154,"Ñ":155,"Ò":156,"Ó":157,"Ô":158,"Õ":159,"Ö":160,"×":161,"Ø":162,"Ù":163,"Ú":164,"Û":165,"Ü":166,"Ý":167,"Þ":168,"ß":169,"à":170,"á":171,"â":172,"ã":173,"ä":174,"å":175,"æ":176,"ç":177,"è":178,"é":179,"ê":180,"ë":181,"ì":182,"í":183,"î":184,"ï":185,"ð":186,"ñ":187,"ò":188,"ó":189,"ô":190,"õ":191,"ö":192,"÷":193,"ø":194,"ù":195,"ú":196,"û":197,"ü":198,"ý":199,"þ":200,"ÿ":201,"Ā":202,"ā":203,"Ă":204,"ă":205,"Ą":206,"ą":207,"Ć":208,"ć":209,"Ĉ":210,"ĉ":211,"Ċ":212,"ċ":213,"Č":214,"č":215,"Ď":216,"ď":217,"Đ":218,"đ":219,"Ē":220,"ē":221,"Ĕ":222,"ĕ":223,"Ė":224,"ė":225,"Ę":226,"ę":227,"Ě":228,"ě":229,"Ĝ":230,"ĝ":231,"Ğ":232,"ğ":233,"Ġ":234,"ġ":235,"Ģ":236,"ģ":237,"Ĥ":238,"ĥ":239,"Ħ":240,"ħ":241,"Ĩ":242,"ĩ":243,"Ī":244,"ī":245,"Ĭ":246,"ĭ":247,"Į":248,"į":249,"İ":250,"ı":251,"IJ":252,"ij":253,"Ĵ":254,"ĵ":255,"Ķ":256,"ķ":257,"ĸ":258,"Ĺ":259,"ĺ":260,"Ļ":261,"ļ":262,"Ľ":263,"ľ":264,"Ŀ":265,"ŀ":266,"Ł":267,"ł":268,"Ń":269,"Ġa":270,"in":271,"ing":272,"ou":273,"th":274,"ve":275,"Ġc":276,"Ġo":277,"Ġs":278,"an":279,"re":280,"Ġl":281,"ea":282,"gh":283,"ri":284,"Ġw":285,"Ġth":286,"ver":287,"Ġthe":288,"ds":289,"igh":290,"nd":291,"on":292,"or":293,"se":294,"st":295,"ti":296,"Ġf":297,"Ġin":298,"Ġri":299,"Ġcl":300,"Ġligh":301,"ati":302,"de":303,"it":304,"mn":305,"mati":306,"ra":307,"ss":308,"ting":309,"ut":310,"Ġb":311,"Ġd":312,"Ġm":313,"Ġre":314,"Ġab":315,"Ġat":316,"Ġand":317,"oun":318,"ouds":319,"Ġon":320,"Ġst":321,"Ġfo":322,"Ġriver":323,"Ġclouds":324,"matic":325,"Ġred":326,"!?":327,"\"/":328,"#$":329,"%&":330,"'\"/":331,"()":332,"*+":333,",:":334,"-()":335,".,:":336,";!?":337,"<>":338,"=<>":339,"@#$":340,"AB":341,"CD":342,"EF":343,"GH":344,"IJ":345,"KL":346,"MN":347,"OP":348,"QR":349,"ST":350,"UV":351,"WX":352,"YZ":353,"[]":354,"\\@#$":355,"^_":356,"`^_":357,"ap":358,"aw":359,"ain":360,"aut":361,"ce":362,"cde":363,"cap":364,"dg":365,"ding":366,"dra":367,"ep":368,"er":369,"et":370,"ematic":371,"ff":372,"fu":373,"fgh":374,"gr":375,"hor":376,"ij":377,"ith":378,"iet":379,"iff":380,"kl":381,"ld":382,"man":383,"no":384,"ning":385,"nse":386,"op":387,"ove":388,"over":389,"oon":390,"oss":391,"oman":392,"por":393,"qr":394,"qu":395,"ring":396,"ron":397,"ross":398,"sh":399,"tra":400,"tain":401,"ter":402,"tning":403,"uv":404,"uti":405,"umn":406,"unse":407,"wx":408,"yz":409,"youn":410,"{}":411,"|~":412,"Ġ.,:":413,"Ġgr":414,"Ġhor":415,"Ġpor":416,"Ġqu":417,"Ġyoun":418,"Ġast":419,"Ġaut":420,"inematic":421,"inter":422,"ound":423,"ouse":424,"thouse":425,"ves":426,"Ġcover":427,"Ġcross":428,"Ġcinematic":429,"Ġof":430,"Ġover":431,"Ġoce":432,"Ġold":433,"Ġsea":434,"Ġsit":435,"Ġsno":436,"Ġsunse":437,"ands":438,"rest":439,"ress":440,"resh":441,"Ġlea":442,"Ġlands":443,"earing":444,"eauti":445,"ridg":446,"Ġwith":447,"Ġwoman":448,"Ġwinter":449,"Ġwearing":450,"one":451,"orm":452,"stuv":453,"Ġfresh":454,"Ġriding":455,"Ġcliff":456,"Ġlighting":457,"Ġlightning":458,"Ġlighthouse":459,"deep":460,"mnop":461,"Ġbeauti":462,"Ġbridg":463,"Ġdaw":464,"Ġdress":465,"Ġmoun":466,"Ġmoon":467,"Ġabcde":468,"Ġabove":469,"Ġstone":470,"Ġstorm":471,"Ġfox":472,"Ġforest":473,"Ġrivers":474,"%&*+":475,"'\"/\\@#$":476,"-()[]":477,";!?-()[]":478,"=<>|~":479,"ABCD":480,"EFGH":481,"IJKL":482,"MNOP":483,"QRST":484,"UVWX":485,"cape":486,"dramatic":487,"ful":488,"fghij":489,"klmnop":490,"qrstuv":491,"ronaut":492,"trait":493,"tains":494,"wxyz":495,"{}'\"/\\@#$":496,"Ġ.,:;!?-()[]":497,"Ġground":498,"Ġhorse":499,"Ġportrait":500,"Ġquiet":501,"Ġyoung":502,"Ġastronaut":503,"Ġautumn":504,"Ġcovering":505,"Ġcrossing":506,"Ġocean":507,"Ġsitting":508,"Ġsnow":509,"Ġsunset":510,"Ġleaves":511,"Ġlandscape":512,"Ġbeautiful":513,"Ġbridge":514,"Ġdawn":515,"Ġmountains":516,"Ġabcdefghij":517,"%&*+=<>|~":518,"ABCDEFGH":519,"IJKLMNOP":520,"QRSTUVWX":521,"klmnopqrstuv":522,"{}'\"/\\@#$%&*+=<>|~":523,"Ġ.,:;!?-()[]{}'\"/\\@#$%&*+=<>|~":524,"Ġabcdefghijklmnopqrstuv":525,"ABCDEFGHIJKLMNOP":526,"QRSTUVWXYZ":527,"Ġ.,:;!?-()[]{}'\"/\\@#$%&*+=<>|~`^_":528,"Ġabcdefghijklmnopqrstuvwxyz":529,"ABCDEFGHIJKLMNOPQRSTUVWXYZ":530}