tokenizer/tokenizer_meta.json

{
  "vocab_size": 16000,
  "special_tokens": {
    "<|bos|>": 0,
    "<|eos|>": 1,
    "<|pad|>": 2,
    "<|sep|>": 3,
    "<|unk|>": 4,
    "<|sys|>": 5,
    "<|usr|>": 6,
    "<|ast|>": 7
  },
  "bos_id": 0,
  "eos_id": 1,
  "pad_id": 2,
  "sep_id": 3,
  "unk_id": 4
}