config.json
22.1 KB · 550 lines · json Raw
1 {
2 "architectures": [
3 "Qwen3_5ForConditionalGeneration"
4 ],
5 "dtype": "bfloat16",
6 "head_dim": 256,
7 "image_token_id": 248056,
8 "language_model_only": false,
9 "model_type": "qwen3_5",
10 "mtp_num_hidden_layers": 1,
11 "num_attention_heads": 24,
12 "num_key_value_heads": 4,
13 "quantization_config": {
14 "config_groups": {
15 "group_0": {
16 "format": "float-quantized",
17 "input_activations": {
18 "actorder": null,
19 "block_structure": null,
20 "dynamic": true,
21 "group_size": null,
22 "num_bits": 8,
23 "observer": null,
24 "observer_kwargs": {},
25 "scale_dtype": null,
26 "strategy": "token",
27 "symmetric": true,
28 "type": "float",
29 "zp_dtype": null
30 },
31 "output_activations": null,
32 "targets": [
33 "re:.*self_attn\\.(q|k|v|o)_proj$",
34 "re:.*linear_attn\\.(in_proj_qkv|in_proj_z|out_proj)$",
35 "re:.*lm_head",
36 "re:.*layers\\.(56|57|58|59|60|61|62|63)\\.mlp\\.(gate|up|down)_proj$"
37 ],
38 "weights": {
39 "actorder": null,
40 "block_structure": null,
41 "dynamic": false,
42 "group_size": null,
43 "num_bits": 8,
44 "observer": "memoryless_minmax",
45 "observer_kwargs": {},
46 "scale_dtype": null,
47 "strategy": "channel",
48 "symmetric": true,
49 "type": "float",
50 "zp_dtype": null
51 }
52 },
53 "group_1": {
54 "format": "nvfp4-pack-quantized",
55 "input_activations": {
56 "actorder": null,
57 "block_structure": null,
58 "dynamic": "local",
59 "group_size": 16,
60 "num_bits": 4,
61 "observer": "static_minmax",
62 "observer_kwargs": {},
63 "scale_dtype": "torch.float8_e4m3fn",
64 "strategy": "tensor_group",
65 "symmetric": true,
66 "type": "float",
67 "zp_dtype": null
68 },
69 "output_activations": null,
70 "targets": [
71 "re:.*mlp\\.(gate|up|down)_proj$"
72 ],
73 "weights": {
74 "actorder": "static",
75 "block_structure": null,
76 "dynamic": false,
77 "group_size": 16,
78 "num_bits": 4,
79 "observer": "imatrix_mse",
80 "observer_kwargs": {},
81 "scale_dtype": "torch.float8_e4m3fn",
82 "strategy": "tensor_group",
83 "symmetric": true,
84 "type": "float",
85 "zp_dtype": null
86 }
87 }
88 },
89 "format": "mixed-precision",
90 "global_compression_ratio": null,
91 "ignore": [
92 "model.visual.blocks.0.attn.qkv",
93 "model.visual.blocks.0.attn.proj",
94 "model.visual.blocks.0.mlp.linear_fc1",
95 "model.visual.blocks.0.mlp.linear_fc2",
96 "model.visual.blocks.1.attn.qkv",
97 "model.visual.blocks.1.attn.proj",
98 "model.visual.blocks.1.mlp.linear_fc1",
99 "model.visual.blocks.1.mlp.linear_fc2",
100 "model.visual.blocks.2.attn.qkv",
101 "model.visual.blocks.2.attn.proj",
102 "model.visual.blocks.2.mlp.linear_fc1",
103 "model.visual.blocks.2.mlp.linear_fc2",
104 "model.visual.blocks.3.attn.qkv",
105 "model.visual.blocks.3.attn.proj",
106 "model.visual.blocks.3.mlp.linear_fc1",
107 "model.visual.blocks.3.mlp.linear_fc2",
108 "model.visual.blocks.4.attn.qkv",
109 "model.visual.blocks.4.attn.proj",
110 "model.visual.blocks.4.mlp.linear_fc1",
111 "model.visual.blocks.4.mlp.linear_fc2",
112 "model.visual.blocks.5.attn.qkv",
113 "model.visual.blocks.5.attn.proj",
114 "model.visual.blocks.5.mlp.linear_fc1",
115 "model.visual.blocks.5.mlp.linear_fc2",
116 "model.visual.blocks.6.attn.qkv",
117 "model.visual.blocks.6.attn.proj",
118 "model.visual.blocks.6.mlp.linear_fc1",
119 "model.visual.blocks.6.mlp.linear_fc2",
120 "model.visual.blocks.7.attn.qkv",
121 "model.visual.blocks.7.attn.proj",
122 "model.visual.blocks.7.mlp.linear_fc1",
123 "model.visual.blocks.7.mlp.linear_fc2",
124 "model.visual.blocks.8.attn.qkv",
125 "model.visual.blocks.8.attn.proj",
126 "model.visual.blocks.8.mlp.linear_fc1",
127 "model.visual.blocks.8.mlp.linear_fc2",
128 "model.visual.blocks.9.attn.qkv",
129 "model.visual.blocks.9.attn.proj",
130 "model.visual.blocks.9.mlp.linear_fc1",
131 "model.visual.blocks.9.mlp.linear_fc2",
132 "model.visual.blocks.10.attn.qkv",
133 "model.visual.blocks.10.attn.proj",
134 "model.visual.blocks.10.mlp.linear_fc1",
135 "model.visual.blocks.10.mlp.linear_fc2",
136 "model.visual.blocks.11.attn.qkv",
137 "model.visual.blocks.11.attn.proj",
138 "model.visual.blocks.11.mlp.linear_fc1",
139 "model.visual.blocks.11.mlp.linear_fc2",
140 "model.visual.blocks.12.attn.qkv",
141 "model.visual.blocks.12.attn.proj",
142 "model.visual.blocks.12.mlp.linear_fc1",
143 "model.visual.blocks.12.mlp.linear_fc2",
144 "model.visual.blocks.13.attn.qkv",
145 "model.visual.blocks.13.attn.proj",
146 "model.visual.blocks.13.mlp.linear_fc1",
147 "model.visual.blocks.13.mlp.linear_fc2",
148 "model.visual.blocks.14.attn.qkv",
149 "model.visual.blocks.14.attn.proj",
150 "model.visual.blocks.14.mlp.linear_fc1",
151 "model.visual.blocks.14.mlp.linear_fc2",
152 "model.visual.blocks.15.attn.qkv",
153 "model.visual.blocks.15.attn.proj",
154 "model.visual.blocks.15.mlp.linear_fc1",
155 "model.visual.blocks.15.mlp.linear_fc2",
156 "model.visual.blocks.16.attn.qkv",
157 "model.visual.blocks.16.attn.proj",
158 "model.visual.blocks.16.mlp.linear_fc1",
159 "model.visual.blocks.16.mlp.linear_fc2",
160 "model.visual.blocks.17.attn.qkv",
161 "model.visual.blocks.17.attn.proj",
162 "model.visual.blocks.17.mlp.linear_fc1",
163 "model.visual.blocks.17.mlp.linear_fc2",
164 "model.visual.blocks.18.attn.qkv",
165 "model.visual.blocks.18.attn.proj",
166 "model.visual.blocks.18.mlp.linear_fc1",
167 "model.visual.blocks.18.mlp.linear_fc2",
168 "model.visual.blocks.19.attn.qkv",
169 "model.visual.blocks.19.attn.proj",
170 "model.visual.blocks.19.mlp.linear_fc1",
171 "model.visual.blocks.19.mlp.linear_fc2",
172 "model.visual.blocks.20.attn.qkv",
173 "model.visual.blocks.20.attn.proj",
174 "model.visual.blocks.20.mlp.linear_fc1",
175 "model.visual.blocks.20.mlp.linear_fc2",
176 "model.visual.blocks.21.attn.qkv",
177 "model.visual.blocks.21.attn.proj",
178 "model.visual.blocks.21.mlp.linear_fc1",
179 "model.visual.blocks.21.mlp.linear_fc2",
180 "model.visual.blocks.22.attn.qkv",
181 "model.visual.blocks.22.attn.proj",
182 "model.visual.blocks.22.mlp.linear_fc1",
183 "model.visual.blocks.22.mlp.linear_fc2",
184 "model.visual.blocks.23.attn.qkv",
185 "model.visual.blocks.23.attn.proj",
186 "model.visual.blocks.23.mlp.linear_fc1",
187 "model.visual.blocks.23.mlp.linear_fc2",
188 "model.visual.blocks.24.attn.qkv",
189 "model.visual.blocks.24.attn.proj",
190 "model.visual.blocks.24.mlp.linear_fc1",
191 "model.visual.blocks.24.mlp.linear_fc2",
192 "model.visual.blocks.25.attn.qkv",
193 "model.visual.blocks.25.attn.proj",
194 "model.visual.blocks.25.mlp.linear_fc1",
195 "model.visual.blocks.25.mlp.linear_fc2",
196 "model.visual.blocks.26.attn.qkv",
197 "model.visual.blocks.26.attn.proj",
198 "model.visual.blocks.26.mlp.linear_fc1",
199 "model.visual.blocks.26.mlp.linear_fc2",
200 "model.visual.merger.linear_fc1",
201 "model.visual.merger.linear_fc2",
202 "model.language_model.layers.0.linear_attn",
203 "model.language_model.layers.0.linear_attn.norm",
204 "model.language_model.layers.0.linear_attn.in_proj_b",
205 "model.language_model.layers.0.linear_attn.in_proj_a",
206 "model.language_model.layers.1.linear_attn",
207 "model.language_model.layers.1.linear_attn.norm",
208 "model.language_model.layers.1.linear_attn.in_proj_b",
209 "model.language_model.layers.1.linear_attn.in_proj_a",
210 "model.language_model.layers.2.linear_attn",
211 "model.language_model.layers.2.linear_attn.norm",
212 "model.language_model.layers.2.linear_attn.in_proj_b",
213 "model.language_model.layers.2.linear_attn.in_proj_a",
214 "model.language_model.layers.4.linear_attn",
215 "model.language_model.layers.4.linear_attn.norm",
216 "model.language_model.layers.4.linear_attn.in_proj_b",
217 "model.language_model.layers.4.linear_attn.in_proj_a",
218 "model.language_model.layers.5.linear_attn",
219 "model.language_model.layers.5.linear_attn.norm",
220 "model.language_model.layers.5.linear_attn.in_proj_b",
221 "model.language_model.layers.5.linear_attn.in_proj_a",
222 "model.language_model.layers.6.linear_attn",
223 "model.language_model.layers.6.linear_attn.norm",
224 "model.language_model.layers.6.linear_attn.in_proj_b",
225 "model.language_model.layers.6.linear_attn.in_proj_a",
226 "model.language_model.layers.8.linear_attn",
227 "model.language_model.layers.8.linear_attn.norm",
228 "model.language_model.layers.8.linear_attn.in_proj_b",
229 "model.language_model.layers.8.linear_attn.in_proj_a",
230 "model.language_model.layers.9.linear_attn",
231 "model.language_model.layers.9.linear_attn.norm",
232 "model.language_model.layers.9.linear_attn.in_proj_b",
233 "model.language_model.layers.9.linear_attn.in_proj_a",
234 "model.language_model.layers.10.linear_attn",
235 "model.language_model.layers.10.linear_attn.norm",
236 "model.language_model.layers.10.linear_attn.in_proj_b",
237 "model.language_model.layers.10.linear_attn.in_proj_a",
238 "model.language_model.layers.12.linear_attn",
239 "model.language_model.layers.12.linear_attn.norm",
240 "model.language_model.layers.12.linear_attn.in_proj_b",
241 "model.language_model.layers.12.linear_attn.in_proj_a",
242 "model.language_model.layers.13.linear_attn",
243 "model.language_model.layers.13.linear_attn.norm",
244 "model.language_model.layers.13.linear_attn.in_proj_b",
245 "model.language_model.layers.13.linear_attn.in_proj_a",
246 "model.language_model.layers.14.linear_attn",
247 "model.language_model.layers.14.linear_attn.norm",
248 "model.language_model.layers.14.linear_attn.in_proj_b",
249 "model.language_model.layers.14.linear_attn.in_proj_a",
250 "model.language_model.layers.16.linear_attn",
251 "model.language_model.layers.16.linear_attn.norm",
252 "model.language_model.layers.16.linear_attn.in_proj_b",
253 "model.language_model.layers.16.linear_attn.in_proj_a",
254 "model.language_model.layers.17.linear_attn",
255 "model.language_model.layers.17.linear_attn.norm",
256 "model.language_model.layers.17.linear_attn.in_proj_b",
257 "model.language_model.layers.17.linear_attn.in_proj_a",
258 "model.language_model.layers.18.linear_attn",
259 "model.language_model.layers.18.linear_attn.norm",
260 "model.language_model.layers.18.linear_attn.in_proj_b",
261 "model.language_model.layers.18.linear_attn.in_proj_a",
262 "model.language_model.layers.20.linear_attn",
263 "model.language_model.layers.20.linear_attn.norm",
264 "model.language_model.layers.20.linear_attn.in_proj_b",
265 "model.language_model.layers.20.linear_attn.in_proj_a",
266 "model.language_model.layers.21.linear_attn",
267 "model.language_model.layers.21.linear_attn.norm",
268 "model.language_model.layers.21.linear_attn.in_proj_b",
269 "model.language_model.layers.21.linear_attn.in_proj_a",
270 "model.language_model.layers.22.linear_attn",
271 "model.language_model.layers.22.linear_attn.norm",
272 "model.language_model.layers.22.linear_attn.in_proj_b",
273 "model.language_model.layers.22.linear_attn.in_proj_a",
274 "model.language_model.layers.24.linear_attn",
275 "model.language_model.layers.24.linear_attn.norm",
276 "model.language_model.layers.24.linear_attn.in_proj_b",
277 "model.language_model.layers.24.linear_attn.in_proj_a",
278 "model.language_model.layers.25.linear_attn",
279 "model.language_model.layers.25.linear_attn.norm",
280 "model.language_model.layers.25.linear_attn.in_proj_b",
281 "model.language_model.layers.25.linear_attn.in_proj_a",
282 "model.language_model.layers.26.linear_attn",
283 "model.language_model.layers.26.linear_attn.norm",
284 "model.language_model.layers.26.linear_attn.in_proj_b",
285 "model.language_model.layers.26.linear_attn.in_proj_a",
286 "model.language_model.layers.28.linear_attn",
287 "model.language_model.layers.28.linear_attn.norm",
288 "model.language_model.layers.28.linear_attn.in_proj_b",
289 "model.language_model.layers.28.linear_attn.in_proj_a",
290 "model.language_model.layers.29.linear_attn",
291 "model.language_model.layers.29.linear_attn.norm",
292 "model.language_model.layers.29.linear_attn.in_proj_b",
293 "model.language_model.layers.29.linear_attn.in_proj_a",
294 "model.language_model.layers.30.linear_attn",
295 "model.language_model.layers.30.linear_attn.norm",
296 "model.language_model.layers.30.linear_attn.in_proj_b",
297 "model.language_model.layers.30.linear_attn.in_proj_a",
298 "model.language_model.layers.32.linear_attn",
299 "model.language_model.layers.32.linear_attn.norm",
300 "model.language_model.layers.32.linear_attn.in_proj_b",
301 "model.language_model.layers.32.linear_attn.in_proj_a",
302 "model.language_model.layers.33.linear_attn",
303 "model.language_model.layers.33.linear_attn.norm",
304 "model.language_model.layers.33.linear_attn.in_proj_b",
305 "model.language_model.layers.33.linear_attn.in_proj_a",
306 "model.language_model.layers.34.linear_attn",
307 "model.language_model.layers.34.linear_attn.norm",
308 "model.language_model.layers.34.linear_attn.in_proj_b",
309 "model.language_model.layers.34.linear_attn.in_proj_a",
310 "model.language_model.layers.36.linear_attn",
311 "model.language_model.layers.36.linear_attn.norm",
312 "model.language_model.layers.36.linear_attn.in_proj_b",
313 "model.language_model.layers.36.linear_attn.in_proj_a",
314 "model.language_model.layers.37.linear_attn",
315 "model.language_model.layers.37.linear_attn.norm",
316 "model.language_model.layers.37.linear_attn.in_proj_b",
317 "model.language_model.layers.37.linear_attn.in_proj_a",
318 "model.language_model.layers.38.linear_attn",
319 "model.language_model.layers.38.linear_attn.norm",
320 "model.language_model.layers.38.linear_attn.in_proj_b",
321 "model.language_model.layers.38.linear_attn.in_proj_a",
322 "model.language_model.layers.40.linear_attn",
323 "model.language_model.layers.40.linear_attn.norm",
324 "model.language_model.layers.40.linear_attn.in_proj_b",
325 "model.language_model.layers.40.linear_attn.in_proj_a",
326 "model.language_model.layers.41.linear_attn",
327 "model.language_model.layers.41.linear_attn.norm",
328 "model.language_model.layers.41.linear_attn.in_proj_b",
329 "model.language_model.layers.41.linear_attn.in_proj_a",
330 "model.language_model.layers.42.linear_attn",
331 "model.language_model.layers.42.linear_attn.norm",
332 "model.language_model.layers.42.linear_attn.in_proj_b",
333 "model.language_model.layers.42.linear_attn.in_proj_a",
334 "model.language_model.layers.44.linear_attn",
335 "model.language_model.layers.44.linear_attn.norm",
336 "model.language_model.layers.44.linear_attn.in_proj_b",
337 "model.language_model.layers.44.linear_attn.in_proj_a",
338 "model.language_model.layers.45.linear_attn",
339 "model.language_model.layers.45.linear_attn.norm",
340 "model.language_model.layers.45.linear_attn.in_proj_b",
341 "model.language_model.layers.45.linear_attn.in_proj_a",
342 "model.language_model.layers.46.linear_attn",
343 "model.language_model.layers.46.linear_attn.norm",
344 "model.language_model.layers.46.linear_attn.in_proj_b",
345 "model.language_model.layers.46.linear_attn.in_proj_a",
346 "model.language_model.layers.48.linear_attn",
347 "model.language_model.layers.48.linear_attn.norm",
348 "model.language_model.layers.48.linear_attn.in_proj_b",
349 "model.language_model.layers.48.linear_attn.in_proj_a",
350 "model.language_model.layers.49.linear_attn",
351 "model.language_model.layers.49.linear_attn.norm",
352 "model.language_model.layers.49.linear_attn.in_proj_b",
353 "model.language_model.layers.49.linear_attn.in_proj_a",
354 "model.language_model.layers.50.linear_attn",
355 "model.language_model.layers.50.linear_attn.norm",
356 "model.language_model.layers.50.linear_attn.in_proj_b",
357 "model.language_model.layers.50.linear_attn.in_proj_a",
358 "model.language_model.layers.52.linear_attn",
359 "model.language_model.layers.52.linear_attn.norm",
360 "model.language_model.layers.52.linear_attn.in_proj_b",
361 "model.language_model.layers.52.linear_attn.in_proj_a",
362 "model.language_model.layers.53.linear_attn",
363 "model.language_model.layers.53.linear_attn.norm",
364 "model.language_model.layers.53.linear_attn.in_proj_b",
365 "model.language_model.layers.53.linear_attn.in_proj_a",
366 "model.language_model.layers.54.linear_attn",
367 "model.language_model.layers.54.linear_attn.norm",
368 "model.language_model.layers.54.linear_attn.in_proj_b",
369 "model.language_model.layers.54.linear_attn.in_proj_a",
370 "model.language_model.layers.56.linear_attn",
371 "model.language_model.layers.56.linear_attn.norm",
372 "model.language_model.layers.56.linear_attn.in_proj_b",
373 "model.language_model.layers.56.linear_attn.in_proj_a",
374 "model.language_model.layers.57.linear_attn",
375 "model.language_model.layers.57.linear_attn.norm",
376 "model.language_model.layers.57.linear_attn.in_proj_b",
377 "model.language_model.layers.57.linear_attn.in_proj_a",
378 "model.language_model.layers.58.linear_attn",
379 "model.language_model.layers.58.linear_attn.norm",
380 "model.language_model.layers.58.linear_attn.in_proj_b",
381 "model.language_model.layers.58.linear_attn.in_proj_a",
382 "model.language_model.layers.60.linear_attn",
383 "model.language_model.layers.60.linear_attn.norm",
384 "model.language_model.layers.60.linear_attn.in_proj_b",
385 "model.language_model.layers.60.linear_attn.in_proj_a",
386 "model.language_model.layers.61.linear_attn",
387 "model.language_model.layers.61.linear_attn.norm",
388 "model.language_model.layers.61.linear_attn.in_proj_b",
389 "model.language_model.layers.61.linear_attn.in_proj_a",
390 "model.language_model.layers.62.linear_attn",
391 "model.language_model.layers.62.linear_attn.norm",
392 "model.language_model.layers.62.linear_attn.in_proj_b",
393 "model.language_model.layers.62.linear_attn.in_proj_a",
394 "re:^mtp.*"
395 ],
396 "kv_cache_scheme": {
397 "actorder": null,
398 "block_structure": null,
399 "dynamic": false,
400 "group_size": null,
401 "num_bits": 8,
402 "observer": "static_minmax",
403 "observer_kwargs": {},
404 "scale_dtype": null,
405 "strategy": "tensor",
406 "symmetric": true,
407 "type": "float",
408 "zp_dtype": null
409 },
410 "quant_method": "compressed-tensors",
411 "quantization_status": "compressed",
412 "sparsity_config": {},
413 "transform_config": {},
414 "version": "0.17.2.a20260707"
415 },
416 "text_config": {
417 "attention_bias": false,
418 "attention_dropout": 0.0,
419 "attn_output_gate": true,
420 "bos_token_id": 248044,
421 "dtype": "bfloat16",
422 "eos_token_id": 248044,
423 "full_attention_interval": 4,
424 "head_dim": 256,
425 "hidden_act": "silu",
426 "hidden_size": 5120,
427 "initializer_range": 0.02,
428 "intermediate_size": 17408,
429 "layer_types": [
430 "linear_attention",
431 "linear_attention",
432 "linear_attention",
433 "full_attention",
434 "linear_attention",
435 "linear_attention",
436 "linear_attention",
437 "full_attention",
438 "linear_attention",
439 "linear_attention",
440 "linear_attention",
441 "full_attention",
442 "linear_attention",
443 "linear_attention",
444 "linear_attention",
445 "full_attention",
446 "linear_attention",
447 "linear_attention",
448 "linear_attention",
449 "full_attention",
450 "linear_attention",
451 "linear_attention",
452 "linear_attention",
453 "full_attention",
454 "linear_attention",
455 "linear_attention",
456 "linear_attention",
457 "full_attention",
458 "linear_attention",
459 "linear_attention",
460 "linear_attention",
461 "full_attention",
462 "linear_attention",
463 "linear_attention",
464 "linear_attention",
465 "full_attention",
466 "linear_attention",
467 "linear_attention",
468 "linear_attention",
469 "full_attention",
470 "linear_attention",
471 "linear_attention",
472 "linear_attention",
473 "full_attention",
474 "linear_attention",
475 "linear_attention",
476 "linear_attention",
477 "full_attention",
478 "linear_attention",
479 "linear_attention",
480 "linear_attention",
481 "full_attention",
482 "linear_attention",
483 "linear_attention",
484 "linear_attention",
485 "full_attention",
486 "linear_attention",
487 "linear_attention",
488 "linear_attention",
489 "full_attention",
490 "linear_attention",
491 "linear_attention",
492 "linear_attention",
493 "full_attention"
494 ],
495 "linear_conv_kernel_dim": 4,
496 "linear_key_head_dim": 128,
497 "linear_num_key_heads": 16,
498 "linear_num_value_heads": 48,
499 "linear_value_head_dim": 128,
500 "mamba_ssm_dtype": "float32",
501 "max_position_embeddings": 262144,
502 "model_type": "qwen3_5_text",
503 "mtp_num_hidden_layers": 1,
504 "mtp_use_dedicated_embeddings": false,
505 "num_attention_heads": 24,
506 "num_hidden_layers": 64,
507 "num_key_value_heads": 4,
508 "output_gate_type": "swish",
509 "pad_token_id": null,
510 "partial_rotary_factor": 0.25,
511 "rms_norm_eps": 1e-06,
512 "rope_parameters": {
513 "mrope_interleaved": true,
514 "mrope_section": [
515 11,
516 11,
517 10
518 ],
519 "partial_rotary_factor": 0.25,
520 "rope_theta": 10000000,
521 "rope_type": "default"
522 },
523 "tie_word_embeddings": false,
524 "use_cache": true,
525 "vocab_size": 248320
526 },
527 "tie_word_embeddings": false,
528 "transformers_version": "5.13.0",
529 "unsloth_fixed_mtp": true,
530 "video_token_id": 248057,
531 "vision_config": {
532 "deepstack_visual_indexes": [],
533 "depth": 27,
534 "dtype": "bfloat16",
535 "hidden_act": "gelu_pytorch_tanh",
536 "hidden_size": 1152,
537 "in_channels": 3,
538 "initializer_range": 0.02,
539 "intermediate_size": 4304,
540 "model_type": "qwen3_5_vision",
541 "num_heads": 16,
542 "num_position_embeddings": 2304,
543 "out_hidden_size": 5120,
544 "patch_size": 16,
545 "spatial_merge_size": 2,
546 "temporal_patch_size": 2
547 },
548 "vision_end_token_id": 248054,
549 "vision_start_token_id": 248053
550 }