tokenizer_config.json
113 B · 6 lines · json Raw
1 {
2 "model_max_length": 512,
3 "pad_token": "$",
4 "tokenizer_class": "PreTrainedTokenizer",
5 "unk_token": "$"
6 }