tokenizer_config.json
10.6 KB · 239 lines · json Raw
1 {
2 "add_bos_token": false,
3 "add_prefix_space": false,
4 "added_tokens_decoder": {
5 "151643": {
6 "content": "<|endoftext|>",
7 "lstrip": false,
8 "normalized": false,
9 "rstrip": false,
10 "single_word": false,
11 "special": true
12 },
13 "151644": {
14 "content": "<|im_start|>",
15 "lstrip": false,
16 "normalized": false,
17 "rstrip": false,
18 "single_word": false,
19 "special": true
20 },
21 "151645": {
22 "content": "<|im_end|>",
23 "lstrip": false,
24 "normalized": false,
25 "rstrip": false,
26 "single_word": false,
27 "special": true
28 },
29 "151646": {
30 "content": "<|object_ref_start|>",
31 "lstrip": false,
32 "normalized": false,
33 "rstrip": false,
34 "single_word": false,
35 "special": true
36 },
37 "151647": {
38 "content": "<|object_ref_end|>",
39 "lstrip": false,
40 "normalized": false,
41 "rstrip": false,
42 "single_word": false,
43 "special": true
44 },
45 "151648": {
46 "content": "<|box_start|>",
47 "lstrip": false,
48 "normalized": false,
49 "rstrip": false,
50 "single_word": false,
51 "special": true
52 },
53 "151649": {
54 "content": "<|box_end|>",
55 "lstrip": false,
56 "normalized": false,
57 "rstrip": false,
58 "single_word": false,
59 "special": true
60 },
61 "151650": {
62 "content": "<|quad_start|>",
63 "lstrip": false,
64 "normalized": false,
65 "rstrip": false,
66 "single_word": false,
67 "special": true
68 },
69 "151651": {
70 "content": "<|quad_end|>",
71 "lstrip": false,
72 "normalized": false,
73 "rstrip": false,
74 "single_word": false,
75 "special": true
76 },
77 "151652": {
78 "content": "<|vision_start|>",
79 "lstrip": false,
80 "normalized": false,
81 "rstrip": false,
82 "single_word": false,
83 "special": true
84 },
85 "151653": {
86 "content": "<|vision_end|>",
87 "lstrip": false,
88 "normalized": false,
89 "rstrip": false,
90 "single_word": false,
91 "special": true
92 },
93 "151654": {
94 "content": "<|vision_pad|>",
95 "lstrip": false,
96 "normalized": false,
97 "rstrip": false,
98 "single_word": false,
99 "special": true
100 },
101 "151655": {
102 "content": "<|image_pad|>",
103 "lstrip": false,
104 "normalized": false,
105 "rstrip": false,
106 "single_word": false,
107 "special": true
108 },
109 "151656": {
110 "content": "<|video_pad|>",
111 "lstrip": false,
112 "normalized": false,
113 "rstrip": false,
114 "single_word": false,
115 "special": true
116 },
117 "151657": {
118 "content": "<tool_call>",
119 "lstrip": false,
120 "normalized": false,
121 "rstrip": false,
122 "single_word": false,
123 "special": false
124 },
125 "151658": {
126 "content": "</tool_call>",
127 "lstrip": false,
128 "normalized": false,
129 "rstrip": false,
130 "single_word": false,
131 "special": false
132 },
133 "151659": {
134 "content": "<|fim_prefix|>",
135 "lstrip": false,
136 "normalized": false,
137 "rstrip": false,
138 "single_word": false,
139 "special": false
140 },
141 "151660": {
142 "content": "<|fim_middle|>",
143 "lstrip": false,
144 "normalized": false,
145 "rstrip": false,
146 "single_word": false,
147 "special": false
148 },
149 "151661": {
150 "content": "<|fim_suffix|>",
151 "lstrip": false,
152 "normalized": false,
153 "rstrip": false,
154 "single_word": false,
155 "special": false
156 },
157 "151662": {
158 "content": "<|fim_pad|>",
159 "lstrip": false,
160 "normalized": false,
161 "rstrip": false,
162 "single_word": false,
163 "special": false
164 },
165 "151663": {
166 "content": "<|repo_name|>",
167 "lstrip": false,
168 "normalized": false,
169 "rstrip": false,
170 "single_word": false,
171 "special": false
172 },
173 "151664": {
174 "content": "<|file_sep|>",
175 "lstrip": false,
176 "normalized": false,
177 "rstrip": false,
178 "single_word": false,
179 "special": false
180 },
181 "151665": {
182 "content": "<tool_response>",
183 "lstrip": false,
184 "normalized": false,
185 "rstrip": false,
186 "single_word": false,
187 "special": false
188 },
189 "151666": {
190 "content": "</tool_response>",
191 "lstrip": false,
192 "normalized": false,
193 "rstrip": false,
194 "single_word": false,
195 "special": false
196 },
197 "151667": {
198 "content": "<think>",
199 "lstrip": false,
200 "normalized": false,
201 "rstrip": false,
202 "single_word": false,
203 "special": false
204 },
205 "151668": {
206 "content": "</think>",
207 "lstrip": false,
208 "normalized": false,
209 "rstrip": false,
210 "single_word": false,
211 "special": false
212 }
213 },
214 "additional_special_tokens": [
215 "<|im_start|>",
216 "<|im_end|>",
217 "<|object_ref_start|>",
218 "<|object_ref_end|>",
219 "<|box_start|>",
220 "<|box_end|>",
221 "<|quad_start|>",
222 "<|quad_end|>",
223 "<|vision_start|>",
224 "<|vision_end|>",
225 "<|vision_pad|>",
226 "<|image_pad|>",
227 "<|video_pad|>"
228 ],
229 "bos_token": null,
230 "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {%- if messages[0].content is string %}\n {{- messages[0].content }}\n {%- else %}\n {%- for content in messages[0].content %}\n {%- if 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].content is string %}\n {{- messages[0].content }}\n {%- else %}\n {%- for content in messages[0].content %}\n {%- if 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- for message in messages %}\n {%- if message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content in message.content %}\n {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}\n <|vision_start|><|image_pad|><|vision_end|>\n {%- elif content.type == 'video' or 'video' in content %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}\n <|vision_start|><|video_pad|><|vision_end|>\n {%- elif 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content_item in message.content %}\n {%- if 'text' in content_item %}\n {{- content_item.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and message.content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content in message.content %}\n {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}\n <|vision_start|><|image_pad|><|vision_end|>\n {%- elif content.type == 'video' or 'video' in content %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}\n <|vision_start|><|video_pad|><|vision_end|>\n {%- elif 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
231 "clean_up_tokenization_spaces": false,
232 "eos_token": "<|im_end|>",
233 "errors": "replace",
234 "model_max_length": 262144,
235 "pad_token": "<|endoftext|>",
236 "split_special_tokens": false,
237 "tokenizer_class": "Qwen2Tokenizer",
238 "unk_token": null
239 }