config.json
16.3 KB · 384 lines · json Raw
1 {
2 "architectures": [
3 "Qwen3_5ForConditionalGeneration"
4 ],
5 "image_token_id": 248056,
6 "language_model_only": false,
7 "model_type": "qwen3_5",
8 "quantization_config": {
9 "config_groups": {
10 "group_0": {
11 "format": "pack-quantized",
12 "input_activations": null,
13 "output_activations": null,
14 "targets": [
15 "Linear"
16 ],
17 "weights": {
18 "actorder": null,
19 "block_structure": null,
20 "dynamic": false,
21 "group_size": 32,
22 "num_bits": 4,
23 "observer": "mse",
24 "observer_kwargs": {},
25 "scale_dtype": null,
26 "strategy": "group",
27 "symmetric": false,
28 "type": "int",
29 "zp_dtype": "torch.int8"
30 }
31 }
32 },
33 "format": "pack-quantized",
34 "global_compression_ratio": null,
35 "ignore": [
36 "model.visual.blocks.0.attn.qkv",
37 "model.visual.blocks.0.attn.proj",
38 "model.visual.blocks.0.mlp.linear_fc1",
39 "model.visual.blocks.0.mlp.linear_fc2",
40 "model.visual.blocks.1.attn.qkv",
41 "model.visual.blocks.1.attn.proj",
42 "model.visual.blocks.1.mlp.linear_fc1",
43 "model.visual.blocks.1.mlp.linear_fc2",
44 "model.visual.blocks.2.attn.qkv",
45 "model.visual.blocks.2.attn.proj",
46 "model.visual.blocks.2.mlp.linear_fc1",
47 "model.visual.blocks.2.mlp.linear_fc2",
48 "model.visual.blocks.3.attn.qkv",
49 "model.visual.blocks.3.attn.proj",
50 "model.visual.blocks.3.mlp.linear_fc1",
51 "model.visual.blocks.3.mlp.linear_fc2",
52 "model.visual.blocks.4.attn.qkv",
53 "model.visual.blocks.4.attn.proj",
54 "model.visual.blocks.4.mlp.linear_fc1",
55 "model.visual.blocks.4.mlp.linear_fc2",
56 "model.visual.blocks.5.attn.qkv",
57 "model.visual.blocks.5.attn.proj",
58 "model.visual.blocks.5.mlp.linear_fc1",
59 "model.visual.blocks.5.mlp.linear_fc2",
60 "model.visual.blocks.6.attn.qkv",
61 "model.visual.blocks.6.attn.proj",
62 "model.visual.blocks.6.mlp.linear_fc1",
63 "model.visual.blocks.6.mlp.linear_fc2",
64 "model.visual.blocks.7.attn.qkv",
65 "model.visual.blocks.7.attn.proj",
66 "model.visual.blocks.7.mlp.linear_fc1",
67 "model.visual.blocks.7.mlp.linear_fc2",
68 "model.visual.blocks.8.attn.qkv",
69 "model.visual.blocks.8.attn.proj",
70 "model.visual.blocks.8.mlp.linear_fc1",
71 "model.visual.blocks.8.mlp.linear_fc2",
72 "model.visual.blocks.9.attn.qkv",
73 "model.visual.blocks.9.attn.proj",
74 "model.visual.blocks.9.mlp.linear_fc1",
75 "model.visual.blocks.9.mlp.linear_fc2",
76 "model.visual.blocks.10.attn.qkv",
77 "model.visual.blocks.10.attn.proj",
78 "model.visual.blocks.10.mlp.linear_fc1",
79 "model.visual.blocks.10.mlp.linear_fc2",
80 "model.visual.blocks.11.attn.qkv",
81 "model.visual.blocks.11.attn.proj",
82 "model.visual.blocks.11.mlp.linear_fc1",
83 "model.visual.blocks.11.mlp.linear_fc2",
84 "model.visual.blocks.12.attn.qkv",
85 "model.visual.blocks.12.attn.proj",
86 "model.visual.blocks.12.mlp.linear_fc1",
87 "model.visual.blocks.12.mlp.linear_fc2",
88 "model.visual.blocks.13.attn.qkv",
89 "model.visual.blocks.13.attn.proj",
90 "model.visual.blocks.13.mlp.linear_fc1",
91 "model.visual.blocks.13.mlp.linear_fc2",
92 "model.visual.blocks.14.attn.qkv",
93 "model.visual.blocks.14.attn.proj",
94 "model.visual.blocks.14.mlp.linear_fc1",
95 "model.visual.blocks.14.mlp.linear_fc2",
96 "model.visual.blocks.15.attn.qkv",
97 "model.visual.blocks.15.attn.proj",
98 "model.visual.blocks.15.mlp.linear_fc1",
99 "model.visual.blocks.15.mlp.linear_fc2",
100 "model.visual.blocks.16.attn.qkv",
101 "model.visual.blocks.16.attn.proj",
102 "model.visual.blocks.16.mlp.linear_fc1",
103 "model.visual.blocks.16.mlp.linear_fc2",
104 "model.visual.blocks.17.attn.qkv",
105 "model.visual.blocks.17.attn.proj",
106 "model.visual.blocks.17.mlp.linear_fc1",
107 "model.visual.blocks.17.mlp.linear_fc2",
108 "model.visual.blocks.18.attn.qkv",
109 "model.visual.blocks.18.attn.proj",
110 "model.visual.blocks.18.mlp.linear_fc1",
111 "model.visual.blocks.18.mlp.linear_fc2",
112 "model.visual.blocks.19.attn.qkv",
113 "model.visual.blocks.19.attn.proj",
114 "model.visual.blocks.19.mlp.linear_fc1",
115 "model.visual.blocks.19.mlp.linear_fc2",
116 "model.visual.blocks.20.attn.qkv",
117 "model.visual.blocks.20.attn.proj",
118 "model.visual.blocks.20.mlp.linear_fc1",
119 "model.visual.blocks.20.mlp.linear_fc2",
120 "model.visual.blocks.21.attn.qkv",
121 "model.visual.blocks.21.attn.proj",
122 "model.visual.blocks.21.mlp.linear_fc1",
123 "model.visual.blocks.21.mlp.linear_fc2",
124 "model.visual.blocks.22.attn.qkv",
125 "model.visual.blocks.22.attn.proj",
126 "model.visual.blocks.22.mlp.linear_fc1",
127 "model.visual.blocks.22.mlp.linear_fc2",
128 "model.visual.blocks.23.attn.qkv",
129 "model.visual.blocks.23.attn.proj",
130 "model.visual.blocks.23.mlp.linear_fc1",
131 "model.visual.blocks.23.mlp.linear_fc2",
132 "model.visual.blocks.24.attn.qkv",
133 "model.visual.blocks.24.attn.proj",
134 "model.visual.blocks.24.mlp.linear_fc1",
135 "model.visual.blocks.24.mlp.linear_fc2",
136 "model.visual.blocks.25.attn.qkv",
137 "model.visual.blocks.25.attn.proj",
138 "model.visual.blocks.25.mlp.linear_fc1",
139 "model.visual.blocks.25.mlp.linear_fc2",
140 "model.visual.blocks.26.attn.qkv",
141 "model.visual.blocks.26.attn.proj",
142 "model.visual.blocks.26.mlp.linear_fc1",
143 "model.visual.blocks.26.mlp.linear_fc2",
144 "model.visual.merger.linear_fc1",
145 "model.visual.merger.linear_fc2",
146 "model.language_model.layers.0.linear_attn.in_proj_b",
147 "model.language_model.layers.0.linear_attn.in_proj_a",
148 "model.language_model.layers.1.linear_attn.in_proj_b",
149 "model.language_model.layers.1.linear_attn.in_proj_a",
150 "model.language_model.layers.2.linear_attn.in_proj_b",
151 "model.language_model.layers.2.linear_attn.in_proj_a",
152 "model.language_model.layers.4.linear_attn.in_proj_b",
153 "model.language_model.layers.4.linear_attn.in_proj_a",
154 "model.language_model.layers.5.linear_attn.in_proj_b",
155 "model.language_model.layers.5.linear_attn.in_proj_a",
156 "model.language_model.layers.6.linear_attn.in_proj_b",
157 "model.language_model.layers.6.linear_attn.in_proj_a",
158 "model.language_model.layers.8.linear_attn.in_proj_b",
159 "model.language_model.layers.8.linear_attn.in_proj_a",
160 "model.language_model.layers.9.linear_attn.in_proj_b",
161 "model.language_model.layers.9.linear_attn.in_proj_a",
162 "model.language_model.layers.10.linear_attn.in_proj_b",
163 "model.language_model.layers.10.linear_attn.in_proj_a",
164 "model.language_model.layers.12.linear_attn.in_proj_b",
165 "model.language_model.layers.12.linear_attn.in_proj_a",
166 "model.language_model.layers.13.linear_attn.in_proj_b",
167 "model.language_model.layers.13.linear_attn.in_proj_a",
168 "model.language_model.layers.14.linear_attn.in_proj_b",
169 "model.language_model.layers.14.linear_attn.in_proj_a",
170 "model.language_model.layers.16.linear_attn.in_proj_b",
171 "model.language_model.layers.16.linear_attn.in_proj_a",
172 "model.language_model.layers.17.linear_attn.in_proj_b",
173 "model.language_model.layers.17.linear_attn.in_proj_a",
174 "model.language_model.layers.18.linear_attn.in_proj_b",
175 "model.language_model.layers.18.linear_attn.in_proj_a",
176 "model.language_model.layers.20.linear_attn.in_proj_b",
177 "model.language_model.layers.20.linear_attn.in_proj_a",
178 "model.language_model.layers.21.linear_attn.in_proj_b",
179 "model.language_model.layers.21.linear_attn.in_proj_a",
180 "model.language_model.layers.22.linear_attn.in_proj_b",
181 "model.language_model.layers.22.linear_attn.in_proj_a",
182 "model.language_model.layers.24.linear_attn.in_proj_b",
183 "model.language_model.layers.24.linear_attn.in_proj_a",
184 "model.language_model.layers.25.linear_attn.in_proj_b",
185 "model.language_model.layers.25.linear_attn.in_proj_a",
186 "model.language_model.layers.26.linear_attn.in_proj_b",
187 "model.language_model.layers.26.linear_attn.in_proj_a",
188 "model.language_model.layers.28.linear_attn.in_proj_b",
189 "model.language_model.layers.28.linear_attn.in_proj_a",
190 "model.language_model.layers.29.linear_attn.in_proj_b",
191 "model.language_model.layers.29.linear_attn.in_proj_a",
192 "model.language_model.layers.30.linear_attn.in_proj_b",
193 "model.language_model.layers.30.linear_attn.in_proj_a",
194 "model.language_model.layers.32.linear_attn.in_proj_b",
195 "model.language_model.layers.32.linear_attn.in_proj_a",
196 "model.language_model.layers.33.linear_attn.in_proj_b",
197 "model.language_model.layers.33.linear_attn.in_proj_a",
198 "model.language_model.layers.34.linear_attn.in_proj_b",
199 "model.language_model.layers.34.linear_attn.in_proj_a",
200 "model.language_model.layers.36.linear_attn.in_proj_b",
201 "model.language_model.layers.36.linear_attn.in_proj_a",
202 "model.language_model.layers.37.linear_attn.in_proj_b",
203 "model.language_model.layers.37.linear_attn.in_proj_a",
204 "model.language_model.layers.38.linear_attn.in_proj_b",
205 "model.language_model.layers.38.linear_attn.in_proj_a",
206 "model.language_model.layers.40.linear_attn.in_proj_b",
207 "model.language_model.layers.40.linear_attn.in_proj_a",
208 "model.language_model.layers.41.linear_attn.in_proj_b",
209 "model.language_model.layers.41.linear_attn.in_proj_a",
210 "model.language_model.layers.42.linear_attn.in_proj_b",
211 "model.language_model.layers.42.linear_attn.in_proj_a",
212 "model.language_model.layers.44.linear_attn.in_proj_b",
213 "model.language_model.layers.44.linear_attn.in_proj_a",
214 "model.language_model.layers.45.linear_attn.in_proj_b",
215 "model.language_model.layers.45.linear_attn.in_proj_a",
216 "model.language_model.layers.46.linear_attn.in_proj_b",
217 "model.language_model.layers.46.linear_attn.in_proj_a",
218 "model.language_model.layers.48.linear_attn.in_proj_b",
219 "model.language_model.layers.48.linear_attn.in_proj_a",
220 "model.language_model.layers.49.linear_attn.in_proj_b",
221 "model.language_model.layers.49.linear_attn.in_proj_a",
222 "model.language_model.layers.50.linear_attn.in_proj_b",
223 "model.language_model.layers.50.linear_attn.in_proj_a",
224 "model.language_model.layers.52.linear_attn.in_proj_b",
225 "model.language_model.layers.52.linear_attn.in_proj_a",
226 "model.language_model.layers.53.linear_attn.in_proj_b",
227 "model.language_model.layers.53.linear_attn.in_proj_a",
228 "model.language_model.layers.54.linear_attn.in_proj_b",
229 "model.language_model.layers.54.linear_attn.in_proj_a",
230 "model.language_model.layers.56.linear_attn.in_proj_b",
231 "model.language_model.layers.56.linear_attn.in_proj_a",
232 "model.language_model.layers.57.linear_attn.in_proj_b",
233 "model.language_model.layers.57.linear_attn.in_proj_a",
234 "model.language_model.layers.58.linear_attn.in_proj_b",
235 "model.language_model.layers.58.linear_attn.in_proj_a",
236 "model.language_model.layers.60.linear_attn.in_proj_b",
237 "model.language_model.layers.60.linear_attn.in_proj_a",
238 "model.language_model.layers.61.linear_attn.in_proj_b",
239 "model.language_model.layers.61.linear_attn.in_proj_a",
240 "model.language_model.layers.62.linear_attn.in_proj_b",
241 "model.language_model.layers.62.linear_attn.in_proj_a",
242 "lm_head",
243 "mtp.fc"
244 ],
245 "kv_cache_scheme": null,
246 "quant_method": "compressed-tensors",
247 "quantization_status": "compressed",
248 "sparsity_config": {},
249 "transform_config": {},
250 "version": "0.14.1.dev20+g5d2f568"
251 },
252 "text_config": {
253 "attention_bias": false,
254 "attention_dropout": 0.0,
255 "attn_output_gate": true,
256 "bos_token_id": 248044,
257 "dtype": "float16",
258 "eos_token_id": 248044,
259 "full_attention_interval": 4,
260 "head_dim": 256,
261 "hidden_act": "silu",
262 "hidden_size": 5120,
263 "initializer_range": 0.02,
264 "intermediate_size": 17408,
265 "layer_types": [
266 "linear_attention",
267 "linear_attention",
268 "linear_attention",
269 "full_attention",
270 "linear_attention",
271 "linear_attention",
272 "linear_attention",
273 "full_attention",
274 "linear_attention",
275 "linear_attention",
276 "linear_attention",
277 "full_attention",
278 "linear_attention",
279 "linear_attention",
280 "linear_attention",
281 "full_attention",
282 "linear_attention",
283 "linear_attention",
284 "linear_attention",
285 "full_attention",
286 "linear_attention",
287 "linear_attention",
288 "linear_attention",
289 "full_attention",
290 "linear_attention",
291 "linear_attention",
292 "linear_attention",
293 "full_attention",
294 "linear_attention",
295 "linear_attention",
296 "linear_attention",
297 "full_attention",
298 "linear_attention",
299 "linear_attention",
300 "linear_attention",
301 "full_attention",
302 "linear_attention",
303 "linear_attention",
304 "linear_attention",
305 "full_attention",
306 "linear_attention",
307 "linear_attention",
308 "linear_attention",
309 "full_attention",
310 "linear_attention",
311 "linear_attention",
312 "linear_attention",
313 "full_attention",
314 "linear_attention",
315 "linear_attention",
316 "linear_attention",
317 "full_attention",
318 "linear_attention",
319 "linear_attention",
320 "linear_attention",
321 "full_attention",
322 "linear_attention",
323 "linear_attention",
324 "linear_attention",
325 "full_attention",
326 "linear_attention",
327 "linear_attention",
328 "linear_attention",
329 "full_attention"
330 ],
331 "linear_conv_kernel_dim": 4,
332 "linear_key_head_dim": 128,
333 "linear_num_key_heads": 16,
334 "linear_num_value_heads": 48,
335 "linear_value_head_dim": 128,
336 "mamba_ssm_dtype": "float32",
337 "max_position_embeddings": 262144,
338 "model_type": "qwen3_5_text",
339 "mtp_num_hidden_layers": 1,
340 "mtp_use_dedicated_embeddings": false,
341 "num_attention_heads": 24,
342 "num_hidden_layers": 64,
343 "num_key_value_heads": 4,
344 "output_gate_type": "swish",
345 "pad_token_id": null,
346 "partial_rotary_factor": 0.25,
347 "rms_norm_eps": 1e-06,
348 "rope_parameters": {
349 "mrope_interleaved": true,
350 "mrope_section": [
351 11,
352 11,
353 10
354 ],
355 "partial_rotary_factor": 0.25,
356 "rope_theta": 10000000,
357 "rope_type": "default"
358 },
359 "tie_word_embeddings": false,
360 "use_cache": true,
361 "vocab_size": 248320
362 },
363 "tie_word_embeddings": false,
364 "transformers_version": "4.57.1",
365 "video_token_id": 248057,
366 "vision_config": {
367 "deepstack_visual_indexes": [],
368 "depth": 27,
369 "hidden_act": "gelu_pytorch_tanh",
370 "hidden_size": 1152,
371 "in_channels": 3,
372 "initializer_range": 0.02,
373 "intermediate_size": 4304,
374 "model_type": "qwen3_5",
375 "num_heads": 16,
376 "num_position_embeddings": 2304,
377 "out_hidden_size": 5120,
378 "patch_size": 16,
379 "spatial_merge_size": 2,
380 "temporal_patch_size": 2
381 },
382 "vision_end_token_id": 248054,
383 "vision_start_token_id": 248053
384 }