tokenizer_config.json
223 B · 8 lines · json Raw
1 {
2 "bos_token": "<|startoftext|>",
3 "clean_up_tokenization_spaces": true,
4 "eos_token": "<|endoftext|>",
5 "model_max_length": 1048576,
6 "pad_token": "<pad>",
7 "tokenizer_class": "PreTrainedTokenizerFast"
8 }