Update README.md
Browse files
README.md
CHANGED
|
@@ -89,7 +89,7 @@ Developed specifically for intelligent routing models and Mixture-of-Experts sys
|
|
| 89 |
```python
|
| 90 |
from transformers import AutoTokenizer
|
| 91 |
|
| 92 |
-
tokenizer = AutoTokenizer.from_pretrained("
|
| 93 |
|
| 94 |
text = "<|im_start|>user\n__CODING__ __PYTHON__ Write a merge sort function.<|im_end|>"
|
| 95 |
print(tokenizer.encode(text))
|
|
|
|
| 89 |
```python
|
| 90 |
from transformers import AutoTokenizer
|
| 91 |
|
| 92 |
+
tokenizer = AutoTokenizer.from_pretrained("CMSManhattan/JiRack-Router-Tokenizer-65K")
|
| 93 |
|
| 94 |
text = "<|im_start|>user\n__CODING__ __PYTHON__ Write a merge sort function.<|im_end|>"
|
| 95 |
print(tokenizer.encode(text))
|