tokenizer_config.json
405 B · 17 lines · json Raw
1 {
2 "backend": "tokenizers",
3 "bos_token": "<|begin_of_text|>",
4 "clean_up_tokenization_spaces": true,
5 "eos_token": "<|end_of_text|>",
6 "is_local": false,
7 "model_input_names": [
8 "input_ids",
9 "attention_mask"
10 ],
11 "model_max_length": 131072,
12 "pad_token": "<|end_of_text|>",
13 "processor_class": "HiggsAudioV2Processor",
14 "tokenizer_class": "TokenizersBackend",
15 "trust_remote": true
16 }
17