File size: 936 Bytes
a2a4a3b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | """Chronicle tokenizer: a thin wrapper over a pickled tiktoken encoding."""
import os
import pickle
from functools import lru_cache
class ChronicleTokenizer:
"""131k-vocabulary BPE tokenizer (tiktoken encoding, trained from scratch)."""
def __init__(self, enc, bos_token="<|bos|>"):
self.enc = enc
self.bos_token_id = self.encode_special(bos_token)
@classmethod
def from_directory(cls, tokenizer_dir):
with open(os.path.join(tokenizer_dir, "tokenizer.pkl"), "rb") as f:
return cls(pickle.load(f))
def get_vocab_size(self):
return self.enc.n_vocab
@lru_cache(maxsize=32)
def encode_special(self, text):
return self.enc.encode_single_token(text)
def get_bos_token_id(self):
return self.bos_token_id
def encode(self, text):
return self.enc.encode_ordinary(text)
def decode(self, ids):
return self.enc.decode(ids)
|