tokenizer_config.json
8.2 KB · 378 lines · json Raw
1 {
2 "added_tokens_decoder": {
3 "0": {
4 "content": "<|pad|>",
5 "lstrip": false,
6 "normalized": false,
7 "rstrip": false,
8 "single_word": false,
9 "special": true
10 },
11 "1": {
12 "content": "<|bos|>",
13 "lstrip": false,
14 "normalized": false,
15 "rstrip": false,
16 "single_word": false,
17 "special": true
18 },
19 "2": {
20 "content": "<|eos|>",
21 "lstrip": false,
22 "normalized": false,
23 "rstrip": false,
24 "single_word": false,
25 "special": true
26 },
27 "3": {
28 "content": "<|TEXT_PROMPT_START|>",
29 "lstrip": false,
30 "normalized": false,
31 "rstrip": false,
32 "single_word": false,
33 "special": true
34 },
35 "4": {
36 "content": "<|TEXT_PROMPT_END|>",
37 "lstrip": false,
38 "normalized": false,
39 "rstrip": false,
40 "single_word": false,
41 "special": true
42 },
43 "5": {
44 "content": "<|SPEECH_GENERATION_START|>",
45 "lstrip": false,
46 "normalized": false,
47 "rstrip": false,
48 "single_word": false,
49 "special": true
50 },
51 "6": {
52 "content": "<|SPEECH_GENERATION_END|>",
53 "lstrip": false,
54 "normalized": false,
55 "rstrip": false,
56 "single_word": false,
57 "special": true
58 },
59 "7": {
60 "content": "<|audio_ref_slot|>",
61 "lstrip": false,
62 "normalized": false,
63 "rstrip": false,
64 "single_word": false,
65 "special": true
66 },
67 "8": {
68 "content": "<|emotion_0|>",
69 "lstrip": false,
70 "normalized": false,
71 "rstrip": false,
72 "single_word": false,
73 "special": true
74 },
75 "9": {
76 "content": "<|emotion_1|>",
77 "lstrip": false,
78 "normalized": false,
79 "rstrip": false,
80 "single_word": false,
81 "special": true
82 },
83 "10": {
84 "content": "<|emotion_2|>",
85 "lstrip": false,
86 "normalized": false,
87 "rstrip": false,
88 "single_word": false,
89 "special": true
90 },
91 "11": {
92 "content": "<|emotion_3|>",
93 "lstrip": false,
94 "normalized": false,
95 "rstrip": false,
96 "single_word": false,
97 "special": true
98 },
99 "12": {
100 "content": "<|emotion_4|>",
101 "lstrip": false,
102 "normalized": false,
103 "rstrip": false,
104 "single_word": false,
105 "special": true
106 },
107 "13": {
108 "content": "<|reserved_0|>",
109 "lstrip": false,
110 "normalized": false,
111 "rstrip": false,
112 "single_word": false,
113 "special": true
114 },
115 "14": {
116 "content": "<|reserved_1|>",
117 "lstrip": false,
118 "normalized": false,
119 "rstrip": false,
120 "single_word": false,
121 "special": true
122 },
123 "15": {
124 "content": "<|reserved_2|>",
125 "lstrip": false,
126 "normalized": false,
127 "rstrip": false,
128 "single_word": false,
129 "special": true
130 },
131 "16": {
132 "content": "<|reserved_3|>",
133 "lstrip": false,
134 "normalized": false,
135 "rstrip": false,
136 "single_word": false,
137 "special": true
138 },
139 "17": {
140 "content": "<|reserved_4|>",
141 "lstrip": false,
142 "normalized": false,
143 "rstrip": false,
144 "single_word": false,
145 "special": true
146 },
147 "18": {
148 "content": "<|reserved_5|>",
149 "lstrip": false,
150 "normalized": false,
151 "rstrip": false,
152 "single_word": false,
153 "special": true
154 },
155 "19": {
156 "content": "<|reserved_6|>",
157 "lstrip": false,
158 "normalized": false,
159 "rstrip": false,
160 "single_word": false,
161 "special": true
162 },
163 "20": {
164 "content": "<|reserved_7|>",
165 "lstrip": false,
166 "normalized": false,
167 "rstrip": false,
168 "single_word": false,
169 "special": true
170 },
171 "21": {
172 "content": "<|reserved_8|>",
173 "lstrip": false,
174 "normalized": false,
175 "rstrip": false,
176 "single_word": false,
177 "special": true
178 },
179 "22": {
180 "content": "<|reserved_9|>",
181 "lstrip": false,
182 "normalized": false,
183 "rstrip": false,
184 "single_word": false,
185 "special": true
186 },
187 "23": {
188 "content": "<|reserved_10|>",
189 "lstrip": false,
190 "normalized": false,
191 "rstrip": false,
192 "single_word": false,
193 "special": true
194 },
195 "24": {
196 "content": "<|reserved_11|>",
197 "lstrip": false,
198 "normalized": false,
199 "rstrip": false,
200 "single_word": false,
201 "special": true
202 },
203 "25": {
204 "content": "<|reserved_12|>",
205 "lstrip": false,
206 "normalized": false,
207 "rstrip": false,
208 "single_word": false,
209 "special": true
210 },
211 "26": {
212 "content": "<|reserved_13|>",
213 "lstrip": false,
214 "normalized": false,
215 "rstrip": false,
216 "single_word": false,
217 "special": true
218 },
219 "27": {
220 "content": "<|reserved_14|>",
221 "lstrip": false,
222 "normalized": false,
223 "rstrip": false,
224 "single_word": false,
225 "special": true
226 },
227 "28": {
228 "content": "<|reserved_15|>",
229 "lstrip": false,
230 "normalized": false,
231 "rstrip": false,
232 "single_word": false,
233 "special": true
234 },
235 "29": {
236 "content": "<|reserved_16|>",
237 "lstrip": false,
238 "normalized": false,
239 "rstrip": false,
240 "single_word": false,
241 "special": true
242 },
243 "30": {
244 "content": "<|reserved_17|>",
245 "lstrip": false,
246 "normalized": false,
247 "rstrip": false,
248 "single_word": false,
249 "special": true
250 },
251 "31": {
252 "content": "<|reserved_18|>",
253 "lstrip": false,
254 "normalized": false,
255 "rstrip": false,
256 "single_word": false,
257 "special": true
258 },
259 "32": {
260 "content": "<|reserved_19|>",
261 "lstrip": false,
262 "normalized": false,
263 "rstrip": false,
264 "single_word": false,
265 "special": true
266 },
267 "33": {
268 "content": "<|reserved_20|>",
269 "lstrip": false,
270 "normalized": false,
271 "rstrip": false,
272 "single_word": false,
273 "special": true
274 },
275 "34": {
276 "content": "<|reserved_21|>",
277 "lstrip": false,
278 "normalized": false,
279 "rstrip": false,
280 "single_word": false,
281 "special": true
282 },
283 "35": {
284 "content": "<|reserved_22|>",
285 "lstrip": false,
286 "normalized": false,
287 "rstrip": false,
288 "single_word": false,
289 "special": true
290 },
291 "36": {
292 "content": "<|reserved_23|>",
293 "lstrip": false,
294 "normalized": false,
295 "rstrip": false,
296 "single_word": false,
297 "special": true
298 },
299 "37": {
300 "content": "<|reserved_24|>",
301 "lstrip": false,
302 "normalized": false,
303 "rstrip": false,
304 "single_word": false,
305 "special": true
306 },
307 "38": {
308 "content": "<|reserved_25|>",
309 "lstrip": false,
310 "normalized": false,
311 "rstrip": false,
312 "single_word": false,
313 "special": true
314 },
315 "39": {
316 "content": "<|reserved_26|>",
317 "lstrip": false,
318 "normalized": false,
319 "rstrip": false,
320 "single_word": false,
321 "special": true
322 },
323 "40": {
324 "content": "<|reserved_27|>",
325 "lstrip": false,
326 "normalized": false,
327 "rstrip": false,
328 "single_word": false,
329 "special": true
330 },
331 "41": {
332 "content": "<|reserved_28|>",
333 "lstrip": false,
334 "normalized": false,
335 "rstrip": false,
336 "single_word": false,
337 "special": true
338 },
339 "42": {
340 "content": "<|reserved_29|>",
341 "lstrip": false,
342 "normalized": false,
343 "rstrip": false,
344 "single_word": false,
345 "special": true
346 },
347 "43": {
348 "content": "<|unk|>",
349 "lstrip": false,
350 "normalized": false,
351 "rstrip": false,
352 "single_word": false,
353 "special": true
354 }
355 },
356 "additional_special_tokens": [
357 "<|TEXT_PROMPT_START|>",
358 "<|TEXT_PROMPT_END|>",
359 "<|SPEECH_GENERATION_START|>",
360 "<|SPEECH_GENERATION_END|>",
361 "<|audio_ref_slot|>",
362 "<|emotion_0|>",
363 "<|emotion_1|>",
364 "<|emotion_2|>",
365 "<|emotion_3|>",
366 "<|emotion_4|>"
367 ],
368 "bos_token": "<|bos|>",
369 "clean_up_tokenization_spaces": false,
370 "eos_token": "<|eos|>",
371 "extra_special_tokens": {},
372 "model_max_length": 2048,
373 "pad_token": "<|pad|>",
374 "padding_side": "right",
375 "tokenizer_class": "PreTrainedTokenizerFast",
376 "unk_token": "<|unk|>"
377 }
378