mikasenghaas commited on
Commit
ea507b8
·
unverified ·
1 Parent(s): dd6752e

Swap special tokens

Browse files
Files changed (2) hide show
  1. tokenizer.json +2 -2
  2. tokenizer_config.json +3 -3
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9340665016419c825c4bdabbcc9acc43b7ca2c68ce142724afa829abb1be5efd
3
- size 19970699
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d9ca0722beb9d6a6e0d0971d3f1b521bb352218624c4111ead24a3dd95e3958c
3
+ size 19970712
tokenizer_config.json CHANGED
@@ -33,7 +33,7 @@
33
  "special": true
34
  },
35
  "151333": {
36
- "content": "<sop>",
37
  "lstrip": false,
38
  "normalized": false,
39
  "rstrip": false,
@@ -41,7 +41,7 @@
41
  "special": true
42
  },
43
  "151334": {
44
- "content": "<eop>",
45
  "lstrip": false,
46
  "normalized": false,
47
  "rstrip": false,
@@ -322,4 +322,4 @@
322
  "padding_side": "left",
323
  "remove_space": false,
324
  "tokenizer_class": "PreTrainedTokenizer"
325
- }
 
33
  "special": true
34
  },
35
  "151333": {
36
+ "content": "<|im_start|>",
37
  "lstrip": false,
38
  "normalized": false,
39
  "rstrip": false,
 
41
  "special": true
42
  },
43
  "151334": {
44
+ "content": "<|im_end|>",
45
  "lstrip": false,
46
  "normalized": false,
47
  "rstrip": false,
 
322
  "padding_side": "left",
323
  "remove_space": false,
324
  "tokenizer_class": "PreTrainedTokenizer"
325
+ }