tokenizer_config.json
| 1 | { |
| 2 | "add_prefix_space": false, |
| 3 | "backend": "tokenizers", |
| 4 | "bos_token": null, |
| 5 | "clean_up_tokenization_spaces": false, |
| 6 | "eos_token": "<|endoftext|>", |
| 7 | "errors": "replace", |
| 8 | "extra_special_tokens": [ |
| 9 | "<|asr|>", |
| 10 | "<|streaming_asr|>", |
| 11 | "<|tts|>", |
| 12 | "<|streaming_tts|>", |
| 13 | "<|audio_cont_txt|>", |
| 14 | "<|audio|>", |
| 15 | "<|audio_end|>", |
| 16 | "<|text|>", |
| 17 | "<|text_end|>", |
| 18 | "<|eoc|>", |
| 19 | "<|user|>", |
| 20 | "<|assistant|>", |
| 21 | "<|system|>", |
| 22 | "<|await_audio|>", |
| 23 | "<|ref_audio|>", |
| 24 | "<|ref_text|>", |
| 25 | "<|emotion:elation|>", |
| 26 | "<|emotion:amusement|>", |
| 27 | "<|emotion:enthusiasm|>", |
| 28 | "<|emotion:determination|>", |
| 29 | "<|emotion:pride|>", |
| 30 | "<|emotion:contentment|>", |
| 31 | "<|emotion:affection|>", |
| 32 | "<|emotion:relief|>", |
| 33 | "<|emotion:contemplation|>", |
| 34 | "<|emotion:confusion|>", |
| 35 | "<|emotion:surprise|>", |
| 36 | "<|emotion:awe|>", |
| 37 | "<|emotion:longing|>", |
| 38 | "<|emotion:arousal|>", |
| 39 | "<|emotion:anger|>", |
| 40 | "<|emotion:fear|>", |
| 41 | "<|emotion:disgust|>", |
| 42 | "<|emotion:bitterness|>", |
| 43 | "<|emotion:sadness|>", |
| 44 | "<|emotion:shame|>", |
| 45 | "<|emotion:helplessness|>", |
| 46 | "<|env:music|>", |
| 47 | "<|env:noise|>", |
| 48 | "<|style:singing|>", |
| 49 | "<|style:shouting|>", |
| 50 | "<|style:whispering|>", |
| 51 | "<|sfx:cough|>", |
| 52 | "<|sfx:laughter|>", |
| 53 | "<|sfx:crying|>", |
| 54 | "<|sfx:screaming|>", |
| 55 | "<|sfx:burping|>", |
| 56 | "<|sfx:humming|>", |
| 57 | "<|sfx:sigh|>", |
| 58 | "<|sfx:sniff|>", |
| 59 | "<|sfx:sneeze|>", |
| 60 | "<|prosody:speed_very_slow|>", |
| 61 | "<|prosody:speed_slow|>", |
| 62 | "<|prosody:speed_fast|>", |
| 63 | "<|prosody:speed_very_fast|>", |
| 64 | "<|prosody:pitch_low|>", |
| 65 | "<|prosody:pitch_high|>", |
| 66 | "<|prosody:pause|>", |
| 67 | "<|prosody:long_pause|>", |
| 68 | "<|chatml|>", |
| 69 | "<|prosody:expressive_high|>", |
| 70 | "<|prosody:expressive_low|>" |
| 71 | ], |
| 72 | "is_local": true, |
| 73 | "model_max_length": 131072, |
| 74 | "pad_token": "<|endoftext|>", |
| 75 | "split_special_tokens": false, |
| 76 | "tokenizer_class": "Qwen2Tokenizer", |
| 77 | "unk_token": null |
| 78 | } |
| 79 | |