config.json
67.8 KB · 1811 lines · json Raw
1 {
2 "architectures": [
3 "Glm5NextForConditionalGeneration"
4 ],
5 "text_config": {
6 "attention_bias": false,
7 "attention_dropout": 0.0,
8 "dtype": "bfloat16",
9 "eos_token_id": [
10 154820,
11 154827,
12 154829
13 ],
14 "first_k_dense_replace": 3,
15 "hc_eps": 1e-06,
16 "hc_mult": 4,
17 "hc_sinkhorn_iters": 20,
18 "head_dim": 0,
19 "hidden_act": "silu",
20 "hidden_size": 4096,
21 "index_head_dim": 128,
22 "index_kpool": 4,
23 "index_kpool_always_select_tail": true,
24 "index_kpool_compress": true,
25 "index_n_heads": 32,
26 "index_topk": 2048,
27 "index_share_for_mtp_iteration": true,
28 "indexer_rope_interleave": true,
29 "indexer_types": [
30 "full",
31 "full",
32 "full",
33 "full",
34 "full",
35 "full",
36 "full",
37 "full",
38 "full",
39 "full",
40 "full",
41 "full",
42 "full",
43 "full",
44 "full",
45 "full",
46 "full",
47 "full",
48 "full",
49 "full",
50 "full",
51 "full",
52 "full",
53 "full",
54 "full",
55 "full",
56 "full",
57 "full",
58 "full",
59 "full",
60 "full",
61 "full",
62 "full",
63 "full",
64 "full",
65 "full",
66 "full",
67 "full",
68 "full",
69 "full",
70 "full",
71 "full",
72 "full",
73 "full",
74 "full"
75 ],
76 "initializer_range": 0.02,
77 "intermediate_size": 12288,
78 "kv_lora_rank": 512,
79 "layer_types": [
80 "linear_attention",
81 "linear_attention",
82 "linear_attention",
83 "deepseek_sparse_attention",
84 "linear_attention",
85 "linear_attention",
86 "linear_attention",
87 "deepseek_sparse_attention",
88 "linear_attention",
89 "linear_attention",
90 "linear_attention",
91 "deepseek_sparse_attention",
92 "linear_attention",
93 "linear_attention",
94 "linear_attention",
95 "deepseek_sparse_attention",
96 "linear_attention",
97 "linear_attention",
98 "linear_attention",
99 "deepseek_sparse_attention",
100 "linear_attention",
101 "linear_attention",
102 "linear_attention",
103 "deepseek_sparse_attention",
104 "linear_attention",
105 "linear_attention",
106 "linear_attention",
107 "deepseek_sparse_attention",
108 "linear_attention",
109 "linear_attention",
110 "linear_attention",
111 "deepseek_sparse_attention",
112 "linear_attention",
113 "linear_attention",
114 "linear_attention",
115 "deepseek_sparse_attention",
116 "linear_attention",
117 "linear_attention",
118 "linear_attention",
119 "deepseek_sparse_attention",
120 "linear_attention",
121 "linear_attention",
122 "linear_attention",
123 "deepseek_sparse_attention",
124 "linear_attention"
125 ],
126 "linear_attn_config": {
127 "num_heads": 64,
128 "gate_lower_bound": -5.0,
129 "head_dim": 128,
130 "short_conv_kernel_size": 4,
131 "kda_layers": [
132 0,
133 1,
134 2,
135 4,
136 5,
137 6,
138 8,
139 9,
140 10,
141 12,
142 13,
143 14,
144 16,
145 17,
146 18,
147 20,
148 21,
149 22,
150 24,
151 25,
152 26,
153 28,
154 29,
155 30,
156 32,
157 33,
158 34,
159 36,
160 37,
161 38,
162 40,
163 41,
164 42,
165 44
166 ],
167 "full_attn_layers": [
168 3,
169 7,
170 11,
171 15,
172 19,
173 23,
174 27,
175 31,
176 35,
177 39,
178 43
179 ]
180 },
181 "max_position_embeddings": 1048576,
182 "mhc": true,
183 "mla_use_nope": true,
184 "mlp_layer_types": [
185 "dense",
186 "dense",
187 "dense",
188 "sparse",
189 "sparse",
190 "sparse",
191 "sparse",
192 "sparse",
193 "sparse",
194 "sparse",
195 "sparse",
196 "sparse",
197 "sparse",
198 "sparse",
199 "sparse",
200 "sparse",
201 "sparse",
202 "sparse",
203 "sparse",
204 "sparse",
205 "sparse",
206 "sparse",
207 "sparse",
208 "sparse",
209 "sparse",
210 "sparse",
211 "sparse",
212 "sparse",
213 "sparse",
214 "sparse",
215 "sparse",
216 "sparse",
217 "sparse",
218 "sparse",
219 "sparse",
220 "sparse",
221 "sparse",
222 "sparse",
223 "sparse",
224 "sparse",
225 "sparse",
226 "sparse",
227 "sparse",
228 "sparse",
229 "sparse"
230 ],
231 "model_type": "glm5_next_text",
232 "moe_intermediate_size": 2048,
233 "moe_router_dtype": "float32",
234 "n_group": 1,
235 "n_routed_experts": 288,
236 "n_shared_experts": 1,
237 "norm_topk_prob": true,
238 "num_attention_heads": 64,
239 "num_experts_per_tok": 8,
240 "num_hidden_layers": 45,
241 "num_key_value_heads": 64,
242 "num_nextn_predict_layers": 1,
243 "output_router_logits": false,
244 "pad_token_id": 154820,
245 "q_lora_rank": 1536,
246 "qk_head_dim": 256,
247 "qk_nope_head_dim": 256,
248 "qk_rope_head_dim": 0,
249 "rms_norm_eps": 1e-05,
250 "routed_scaling_factor": 2.5,
251 "router_aux_loss_coef": 0.001,
252 "scoring_func": "sigmoid",
253 "swiglu_limit": 10.0,
254 "tie_word_embeddings": false,
255 "topk_group": 1,
256 "topk_method": "noaux_tc",
257 "use_cache": true,
258 "v_head_dim": 256,
259 "vocab_size": 154880
260 },
261 "vision_config": {
262 "attention_bias": true,
263 "attention_dropout": 0.0,
264 "depth": 24,
265 "hidden_act": "silu",
266 "hidden_size": 1024,
267 "image_size": 448,
268 "in_channels": 3,
269 "initializer_range": 0.02,
270 "intermediate_size": 4096,
271 "model_type": "glm5_next_vision",
272 "num_heads": 16,
273 "out_hidden_size": 4096,
274 "patch_size": 14,
275 "projection_intermediate_size": 10240,
276 "rms_norm_eps": 1e-05,
277 "spatial_merge_size": 2,
278 "swiglu_limit": 10.0,
279 "temporal_patch_size": 2
280 },
281 "image_token_id": 154854,
282 "video_token_id": 154855,
283 "image_start_token_id": 154830,
284 "image_end_token_id": 154831,
285 "video_start_token_id": 154832,
286 "video_end_token_id": 154833,
287 "tie_word_embeddings": false,
288 "model_type": "glm5_next",
289 "transformers_version": "5.16.0",
290 "quantization_config": {
291 "activation_scheme": "dynamic",
292 "fmt": "e4m3",
293 "modules_to_not_convert": [
294 "attn_mha",
295 "attn_mqa",
296 "dt_bias",
297 "hyper_connection",
298 "lm_head",
299 "mapping_proj",
300 "model.embed_tokens",
301 "model.layers.0.hc_attn_base",
302 "model.layers.0.hc_attn_fn",
303 "model.layers.0.hc_attn_scale",
304 "model.layers.0.hc_ffn_base",
305 "model.layers.0.hc_ffn_fn",
306 "model.layers.0.hc_ffn_scale",
307 "model.layers.0.input_layernorm",
308 "model.layers.0.post_attention_layernorm",
309 "model.layers.0.self_attn.A_log",
310 "model.layers.0.self_attn.b_proj",
311 "model.layers.0.self_attn.dt_bias",
312 "model.layers.0.self_attn.f_a_proj",
313 "model.layers.0.self_attn.f_b_proj",
314 "model.layers.0.self_attn.fused_qkvbfg_a_proj",
315 "model.layers.0.self_attn.g_a_proj",
316 "model.layers.0.self_attn.g_b_proj",
317 "model.layers.0.self_attn.k_conv1d",
318 "model.layers.0.self_attn.k_proj",
319 "model.layers.0.self_attn.o_norm",
320 "model.layers.0.self_attn.o_proj",
321 "model.layers.0.self_attn.q_conv1d",
322 "model.layers.0.self_attn.q_proj",
323 "model.layers.0.self_attn.qkv_proj",
324 "model.layers.0.self_attn.v_conv1d",
325 "model.layers.0.self_attn.v_proj",
326 "model.layers.1.hc_attn_base",
327 "model.layers.1.hc_attn_fn",
328 "model.layers.1.hc_attn_scale",
329 "model.layers.1.hc_ffn_base",
330 "model.layers.1.hc_ffn_fn",
331 "model.layers.1.hc_ffn_scale",
332 "model.layers.1.input_layernorm",
333 "model.layers.1.post_attention_layernorm",
334 "model.layers.1.self_attn.A_log",
335 "model.layers.1.self_attn.b_proj",
336 "model.layers.1.self_attn.dt_bias",
337 "model.layers.1.self_attn.f_a_proj",
338 "model.layers.1.self_attn.f_b_proj",
339 "model.layers.1.self_attn.fused_qkvbfg_a_proj",
340 "model.layers.1.self_attn.g_a_proj",
341 "model.layers.1.self_attn.g_b_proj",
342 "model.layers.1.self_attn.k_conv1d",
343 "model.layers.1.self_attn.k_proj",
344 "model.layers.1.self_attn.o_norm",
345 "model.layers.1.self_attn.o_proj",
346 "model.layers.1.self_attn.q_conv1d",
347 "model.layers.1.self_attn.q_proj",
348 "model.layers.1.self_attn.qkv_proj",
349 "model.layers.1.self_attn.v_conv1d",
350 "model.layers.1.self_attn.v_proj",
351 "model.layers.10.hc_attn_base",
352 "model.layers.10.hc_attn_fn",
353 "model.layers.10.hc_attn_scale",
354 "model.layers.10.hc_ffn_base",
355 "model.layers.10.hc_ffn_fn",
356 "model.layers.10.hc_ffn_scale",
357 "model.layers.10.input_layernorm",
358 "model.layers.10.mlp.gate",
359 "model.layers.10.mlp.gate.e_score_correction_bias",
360 "model.layers.10.post_attention_layernorm",
361 "model.layers.10.self_attn.A_log",
362 "model.layers.10.self_attn.b_proj",
363 "model.layers.10.self_attn.dt_bias",
364 "model.layers.10.self_attn.f_a_proj",
365 "model.layers.10.self_attn.f_b_proj",
366 "model.layers.10.self_attn.fused_qkvbfg_a_proj",
367 "model.layers.10.self_attn.g_a_proj",
368 "model.layers.10.self_attn.g_b_proj",
369 "model.layers.10.self_attn.k_conv1d",
370 "model.layers.10.self_attn.k_proj",
371 "model.layers.10.self_attn.o_norm",
372 "model.layers.10.self_attn.o_proj",
373 "model.layers.10.self_attn.q_conv1d",
374 "model.layers.10.self_attn.q_proj",
375 "model.layers.10.self_attn.qkv_proj",
376 "model.layers.10.self_attn.v_conv1d",
377 "model.layers.10.self_attn.v_proj",
378 "model.layers.11.hc_attn_base",
379 "model.layers.11.hc_attn_fn",
380 "model.layers.11.hc_attn_scale",
381 "model.layers.11.hc_ffn_base",
382 "model.layers.11.hc_ffn_fn",
383 "model.layers.11.hc_ffn_scale",
384 "model.layers.11.input_layernorm",
385 "model.layers.11.mlp.gate",
386 "model.layers.11.mlp.gate.e_score_correction_bias",
387 "model.layers.11.post_attention_layernorm",
388 "model.layers.11.self_attn.indexer.index_kpool_compress_ape",
389 "model.layers.11.self_attn.indexer.index_kpool_compress_gate",
390 "model.layers.11.self_attn.indexer.k_norm",
391 "model.layers.11.self_attn.indexer.k_norm.bias",
392 "model.layers.11.self_attn.indexer.weights_proj",
393 "model.layers.11.self_attn.indexer.wk",
394 "model.layers.11.self_attn.indexer.wq_b",
395 "model.layers.11.self_attn.kv_a_layernorm",
396 "model.layers.11.self_attn.kv_b_proj",
397 "model.layers.11.self_attn.q_a_layernorm",
398 "model.layers.12.hc_attn_base",
399 "model.layers.12.hc_attn_fn",
400 "model.layers.12.hc_attn_scale",
401 "model.layers.12.hc_ffn_base",
402 "model.layers.12.hc_ffn_fn",
403 "model.layers.12.hc_ffn_scale",
404 "model.layers.12.input_layernorm",
405 "model.layers.12.mlp.gate",
406 "model.layers.12.mlp.gate.e_score_correction_bias",
407 "model.layers.12.post_attention_layernorm",
408 "model.layers.12.self_attn.A_log",
409 "model.layers.12.self_attn.b_proj",
410 "model.layers.12.self_attn.dt_bias",
411 "model.layers.12.self_attn.f_a_proj",
412 "model.layers.12.self_attn.f_b_proj",
413 "model.layers.12.self_attn.fused_qkvbfg_a_proj",
414 "model.layers.12.self_attn.g_a_proj",
415 "model.layers.12.self_attn.g_b_proj",
416 "model.layers.12.self_attn.k_conv1d",
417 "model.layers.12.self_attn.k_proj",
418 "model.layers.12.self_attn.o_norm",
419 "model.layers.12.self_attn.o_proj",
420 "model.layers.12.self_attn.q_conv1d",
421 "model.layers.12.self_attn.q_proj",
422 "model.layers.12.self_attn.qkv_proj",
423 "model.layers.12.self_attn.v_conv1d",
424 "model.layers.12.self_attn.v_proj",
425 "model.layers.13.hc_attn_base",
426 "model.layers.13.hc_attn_fn",
427 "model.layers.13.hc_attn_scale",
428 "model.layers.13.hc_ffn_base",
429 "model.layers.13.hc_ffn_fn",
430 "model.layers.13.hc_ffn_scale",
431 "model.layers.13.input_layernorm",
432 "model.layers.13.mlp.gate",
433 "model.layers.13.mlp.gate.e_score_correction_bias",
434 "model.layers.13.post_attention_layernorm",
435 "model.layers.13.self_attn.A_log",
436 "model.layers.13.self_attn.b_proj",
437 "model.layers.13.self_attn.dt_bias",
438 "model.layers.13.self_attn.f_a_proj",
439 "model.layers.13.self_attn.f_b_proj",
440 "model.layers.13.self_attn.fused_qkvbfg_a_proj",
441 "model.layers.13.self_attn.g_a_proj",
442 "model.layers.13.self_attn.g_b_proj",
443 "model.layers.13.self_attn.k_conv1d",
444 "model.layers.13.self_attn.k_proj",
445 "model.layers.13.self_attn.o_norm",
446 "model.layers.13.self_attn.o_proj",
447 "model.layers.13.self_attn.q_conv1d",
448 "model.layers.13.self_attn.q_proj",
449 "model.layers.13.self_attn.qkv_proj",
450 "model.layers.13.self_attn.v_conv1d",
451 "model.layers.13.self_attn.v_proj",
452 "model.layers.14.hc_attn_base",
453 "model.layers.14.hc_attn_fn",
454 "model.layers.14.hc_attn_scale",
455 "model.layers.14.hc_ffn_base",
456 "model.layers.14.hc_ffn_fn",
457 "model.layers.14.hc_ffn_scale",
458 "model.layers.14.input_layernorm",
459 "model.layers.14.mlp.gate",
460 "model.layers.14.mlp.gate.e_score_correction_bias",
461 "model.layers.14.post_attention_layernorm",
462 "model.layers.14.self_attn.A_log",
463 "model.layers.14.self_attn.b_proj",
464 "model.layers.14.self_attn.dt_bias",
465 "model.layers.14.self_attn.f_a_proj",
466 "model.layers.14.self_attn.f_b_proj",
467 "model.layers.14.self_attn.fused_qkvbfg_a_proj",
468 "model.layers.14.self_attn.g_a_proj",
469 "model.layers.14.self_attn.g_b_proj",
470 "model.layers.14.self_attn.k_conv1d",
471 "model.layers.14.self_attn.k_proj",
472 "model.layers.14.self_attn.o_norm",
473 "model.layers.14.self_attn.o_proj",
474 "model.layers.14.self_attn.q_conv1d",
475 "model.layers.14.self_attn.q_proj",
476 "model.layers.14.self_attn.qkv_proj",
477 "model.layers.14.self_attn.v_conv1d",
478 "model.layers.14.self_attn.v_proj",
479 "model.layers.15.hc_attn_base",
480 "model.layers.15.hc_attn_fn",
481 "model.layers.15.hc_attn_scale",
482 "model.layers.15.hc_ffn_base",
483 "model.layers.15.hc_ffn_fn",
484 "model.layers.15.hc_ffn_scale",
485 "model.layers.15.input_layernorm",
486 "model.layers.15.mlp.gate",
487 "model.layers.15.mlp.gate.e_score_correction_bias",
488 "model.layers.15.post_attention_layernorm",
489 "model.layers.15.self_attn.indexer.index_kpool_compress_ape",
490 "model.layers.15.self_attn.indexer.index_kpool_compress_gate",
491 "model.layers.15.self_attn.indexer.k_norm",
492 "model.layers.15.self_attn.indexer.k_norm.bias",
493 "model.layers.15.self_attn.indexer.weights_proj",
494 "model.layers.15.self_attn.indexer.wk",
495 "model.layers.15.self_attn.indexer.wq_b",
496 "model.layers.15.self_attn.kv_a_layernorm",
497 "model.layers.15.self_attn.kv_b_proj",
498 "model.layers.15.self_attn.q_a_layernorm",
499 "model.layers.16.hc_attn_base",
500 "model.layers.16.hc_attn_fn",
501 "model.layers.16.hc_attn_scale",
502 "model.layers.16.hc_ffn_base",
503 "model.layers.16.hc_ffn_fn",
504 "model.layers.16.hc_ffn_scale",
505 "model.layers.16.input_layernorm",
506 "model.layers.16.mlp.gate",
507 "model.layers.16.mlp.gate.e_score_correction_bias",
508 "model.layers.16.post_attention_layernorm",
509 "model.layers.16.self_attn.A_log",
510 "model.layers.16.self_attn.b_proj",
511 "model.layers.16.self_attn.dt_bias",
512 "model.layers.16.self_attn.f_a_proj",
513 "model.layers.16.self_attn.f_b_proj",
514 "model.layers.16.self_attn.fused_qkvbfg_a_proj",
515 "model.layers.16.self_attn.g_a_proj",
516 "model.layers.16.self_attn.g_b_proj",
517 "model.layers.16.self_attn.k_conv1d",
518 "model.layers.16.self_attn.k_proj",
519 "model.layers.16.self_attn.o_norm",
520 "model.layers.16.self_attn.o_proj",
521 "model.layers.16.self_attn.q_conv1d",
522 "model.layers.16.self_attn.q_proj",
523 "model.layers.16.self_attn.qkv_proj",
524 "model.layers.16.self_attn.v_conv1d",
525 "model.layers.16.self_attn.v_proj",
526 "model.layers.17.hc_attn_base",
527 "model.layers.17.hc_attn_fn",
528 "model.layers.17.hc_attn_scale",
529 "model.layers.17.hc_ffn_base",
530 "model.layers.17.hc_ffn_fn",
531 "model.layers.17.hc_ffn_scale",
532 "model.layers.17.input_layernorm",
533 "model.layers.17.mlp.gate",
534 "model.layers.17.mlp.gate.e_score_correction_bias",
535 "model.layers.17.post_attention_layernorm",
536 "model.layers.17.self_attn.A_log",
537 "model.layers.17.self_attn.b_proj",
538 "model.layers.17.self_attn.dt_bias",
539 "model.layers.17.self_attn.f_a_proj",
540 "model.layers.17.self_attn.f_b_proj",
541 "model.layers.17.self_attn.fused_qkvbfg_a_proj",
542 "model.layers.17.self_attn.g_a_proj",
543 "model.layers.17.self_attn.g_b_proj",
544 "model.layers.17.self_attn.k_conv1d",
545 "model.layers.17.self_attn.k_proj",
546 "model.layers.17.self_attn.o_norm",
547 "model.layers.17.self_attn.o_proj",
548 "model.layers.17.self_attn.q_conv1d",
549 "model.layers.17.self_attn.q_proj",
550 "model.layers.17.self_attn.qkv_proj",
551 "model.layers.17.self_attn.v_conv1d",
552 "model.layers.17.self_attn.v_proj",
553 "model.layers.18.hc_attn_base",
554 "model.layers.18.hc_attn_fn",
555 "model.layers.18.hc_attn_scale",
556 "model.layers.18.hc_ffn_base",
557 "model.layers.18.hc_ffn_fn",
558 "model.layers.18.hc_ffn_scale",
559 "model.layers.18.input_layernorm",
560 "model.layers.18.mlp.gate",
561 "model.layers.18.mlp.gate.e_score_correction_bias",
562 "model.layers.18.post_attention_layernorm",
563 "model.layers.18.self_attn.A_log",
564 "model.layers.18.self_attn.b_proj",
565 "model.layers.18.self_attn.dt_bias",
566 "model.layers.18.self_attn.f_a_proj",
567 "model.layers.18.self_attn.f_b_proj",
568 "model.layers.18.self_attn.fused_qkvbfg_a_proj",
569 "model.layers.18.self_attn.g_a_proj",
570 "model.layers.18.self_attn.g_b_proj",
571 "model.layers.18.self_attn.k_conv1d",
572 "model.layers.18.self_attn.k_proj",
573 "model.layers.18.self_attn.o_norm",
574 "model.layers.18.self_attn.o_proj",
575 "model.layers.18.self_attn.q_conv1d",
576 "model.layers.18.self_attn.q_proj",
577 "model.layers.18.self_attn.qkv_proj",
578 "model.layers.18.self_attn.v_conv1d",
579 "model.layers.18.self_attn.v_proj",
580 "model.layers.19.hc_attn_base",
581 "model.layers.19.hc_attn_fn",
582 "model.layers.19.hc_attn_scale",
583 "model.layers.19.hc_ffn_base",
584 "model.layers.19.hc_ffn_fn",
585 "model.layers.19.hc_ffn_scale",
586 "model.layers.19.input_layernorm",
587 "model.layers.19.mlp.gate",
588 "model.layers.19.mlp.gate.e_score_correction_bias",
589 "model.layers.19.post_attention_layernorm",
590 "model.layers.19.self_attn.indexer.index_kpool_compress_ape",
591 "model.layers.19.self_attn.indexer.index_kpool_compress_gate",
592 "model.layers.19.self_attn.indexer.k_norm",
593 "model.layers.19.self_attn.indexer.k_norm.bias",
594 "model.layers.19.self_attn.indexer.weights_proj",
595 "model.layers.19.self_attn.indexer.wk",
596 "model.layers.19.self_attn.indexer.wq_b",
597 "model.layers.19.self_attn.kv_a_layernorm",
598 "model.layers.19.self_attn.kv_b_proj",
599 "model.layers.19.self_attn.q_a_layernorm",
600 "model.layers.2.hc_attn_base",
601 "model.layers.2.hc_attn_fn",
602 "model.layers.2.hc_attn_scale",
603 "model.layers.2.hc_ffn_base",
604 "model.layers.2.hc_ffn_fn",
605 "model.layers.2.hc_ffn_scale",
606 "model.layers.2.input_layernorm",
607 "model.layers.2.post_attention_layernorm",
608 "model.layers.2.self_attn.A_log",
609 "model.layers.2.self_attn.b_proj",
610 "model.layers.2.self_attn.dt_bias",
611 "model.layers.2.self_attn.f_a_proj",
612 "model.layers.2.self_attn.f_b_proj",
613 "model.layers.2.self_attn.fused_qkvbfg_a_proj",
614 "model.layers.2.self_attn.g_a_proj",
615 "model.layers.2.self_attn.g_b_proj",
616 "model.layers.2.self_attn.k_conv1d",
617 "model.layers.2.self_attn.k_proj",
618 "model.layers.2.self_attn.o_norm",
619 "model.layers.2.self_attn.o_proj",
620 "model.layers.2.self_attn.q_conv1d",
621 "model.layers.2.self_attn.q_proj",
622 "model.layers.2.self_attn.qkv_proj",
623 "model.layers.2.self_attn.v_conv1d",
624 "model.layers.2.self_attn.v_proj",
625 "model.layers.20.hc_attn_base",
626 "model.layers.20.hc_attn_fn",
627 "model.layers.20.hc_attn_scale",
628 "model.layers.20.hc_ffn_base",
629 "model.layers.20.hc_ffn_fn",
630 "model.layers.20.hc_ffn_scale",
631 "model.layers.20.input_layernorm",
632 "model.layers.20.mlp.gate",
633 "model.layers.20.mlp.gate.e_score_correction_bias",
634 "model.layers.20.post_attention_layernorm",
635 "model.layers.20.self_attn.A_log",
636 "model.layers.20.self_attn.b_proj",
637 "model.layers.20.self_attn.dt_bias",
638 "model.layers.20.self_attn.f_a_proj",
639 "model.layers.20.self_attn.f_b_proj",
640 "model.layers.20.self_attn.fused_qkvbfg_a_proj",
641 "model.layers.20.self_attn.g_a_proj",
642 "model.layers.20.self_attn.g_b_proj",
643 "model.layers.20.self_attn.k_conv1d",
644 "model.layers.20.self_attn.k_proj",
645 "model.layers.20.self_attn.o_norm",
646 "model.layers.20.self_attn.o_proj",
647 "model.layers.20.self_attn.q_conv1d",
648 "model.layers.20.self_attn.q_proj",
649 "model.layers.20.self_attn.qkv_proj",
650 "model.layers.20.self_attn.v_conv1d",
651 "model.layers.20.self_attn.v_proj",
652 "model.layers.21.hc_attn_base",
653 "model.layers.21.hc_attn_fn",
654 "model.layers.21.hc_attn_scale",
655 "model.layers.21.hc_ffn_base",
656 "model.layers.21.hc_ffn_fn",
657 "model.layers.21.hc_ffn_scale",
658 "model.layers.21.input_layernorm",
659 "model.layers.21.mlp.gate",
660 "model.layers.21.mlp.gate.e_score_correction_bias",
661 "model.layers.21.post_attention_layernorm",
662 "model.layers.21.self_attn.A_log",
663 "model.layers.21.self_attn.b_proj",
664 "model.layers.21.self_attn.dt_bias",
665 "model.layers.21.self_attn.f_a_proj",
666 "model.layers.21.self_attn.f_b_proj",
667 "model.layers.21.self_attn.fused_qkvbfg_a_proj",
668 "model.layers.21.self_attn.g_a_proj",
669 "model.layers.21.self_attn.g_b_proj",
670 "model.layers.21.self_attn.k_conv1d",
671 "model.layers.21.self_attn.k_proj",
672 "model.layers.21.self_attn.o_norm",
673 "model.layers.21.self_attn.o_proj",
674 "model.layers.21.self_attn.q_conv1d",
675 "model.layers.21.self_attn.q_proj",
676 "model.layers.21.self_attn.qkv_proj",
677 "model.layers.21.self_attn.v_conv1d",
678 "model.layers.21.self_attn.v_proj",
679 "model.layers.22.hc_attn_base",
680 "model.layers.22.hc_attn_fn",
681 "model.layers.22.hc_attn_scale",
682 "model.layers.22.hc_ffn_base",
683 "model.layers.22.hc_ffn_fn",
684 "model.layers.22.hc_ffn_scale",
685 "model.layers.22.input_layernorm",
686 "model.layers.22.mlp.gate",
687 "model.layers.22.mlp.gate.e_score_correction_bias",
688 "model.layers.22.post_attention_layernorm",
689 "model.layers.22.self_attn.A_log",
690 "model.layers.22.self_attn.b_proj",
691 "model.layers.22.self_attn.dt_bias",
692 "model.layers.22.self_attn.f_a_proj",
693 "model.layers.22.self_attn.f_b_proj",
694 "model.layers.22.self_attn.fused_qkvbfg_a_proj",
695 "model.layers.22.self_attn.g_a_proj",
696 "model.layers.22.self_attn.g_b_proj",
697 "model.layers.22.self_attn.k_conv1d",
698 "model.layers.22.self_attn.k_proj",
699 "model.layers.22.self_attn.o_norm",
700 "model.layers.22.self_attn.o_proj",
701 "model.layers.22.self_attn.q_conv1d",
702 "model.layers.22.self_attn.q_proj",
703 "model.layers.22.self_attn.qkv_proj",
704 "model.layers.22.self_attn.v_conv1d",
705 "model.layers.22.self_attn.v_proj",
706 "model.layers.23.hc_attn_base",
707 "model.layers.23.hc_attn_fn",
708 "model.layers.23.hc_attn_scale",
709 "model.layers.23.hc_ffn_base",
710 "model.layers.23.hc_ffn_fn",
711 "model.layers.23.hc_ffn_scale",
712 "model.layers.23.input_layernorm",
713 "model.layers.23.mlp.gate",
714 "model.layers.23.mlp.gate.e_score_correction_bias",
715 "model.layers.23.post_attention_layernorm",
716 "model.layers.23.self_attn.indexer.index_kpool_compress_ape",
717 "model.layers.23.self_attn.indexer.index_kpool_compress_gate",
718 "model.layers.23.self_attn.indexer.k_norm",
719 "model.layers.23.self_attn.indexer.k_norm.bias",
720 "model.layers.23.self_attn.indexer.weights_proj",
721 "model.layers.23.self_attn.indexer.wk",
722 "model.layers.23.self_attn.indexer.wq_b",
723 "model.layers.23.self_attn.kv_a_layernorm",
724 "model.layers.23.self_attn.kv_b_proj",
725 "model.layers.23.self_attn.q_a_layernorm",
726 "model.layers.24.hc_attn_base",
727 "model.layers.24.hc_attn_fn",
728 "model.layers.24.hc_attn_scale",
729 "model.layers.24.hc_ffn_base",
730 "model.layers.24.hc_ffn_fn",
731 "model.layers.24.hc_ffn_scale",
732 "model.layers.24.input_layernorm",
733 "model.layers.24.mlp.gate",
734 "model.layers.24.mlp.gate.e_score_correction_bias",
735 "model.layers.24.post_attention_layernorm",
736 "model.layers.24.self_attn.A_log",
737 "model.layers.24.self_attn.b_proj",
738 "model.layers.24.self_attn.dt_bias",
739 "model.layers.24.self_attn.f_a_proj",
740 "model.layers.24.self_attn.f_b_proj",
741 "model.layers.24.self_attn.fused_qkvbfg_a_proj",
742 "model.layers.24.self_attn.g_a_proj",
743 "model.layers.24.self_attn.g_b_proj",
744 "model.layers.24.self_attn.k_conv1d",
745 "model.layers.24.self_attn.k_proj",
746 "model.layers.24.self_attn.o_norm",
747 "model.layers.24.self_attn.o_proj",
748 "model.layers.24.self_attn.q_conv1d",
749 "model.layers.24.self_attn.q_proj",
750 "model.layers.24.self_attn.qkv_proj",
751 "model.layers.24.self_attn.v_conv1d",
752 "model.layers.24.self_attn.v_proj",
753 "model.layers.25.hc_attn_base",
754 "model.layers.25.hc_attn_fn",
755 "model.layers.25.hc_attn_scale",
756 "model.layers.25.hc_ffn_base",
757 "model.layers.25.hc_ffn_fn",
758 "model.layers.25.hc_ffn_scale",
759 "model.layers.25.input_layernorm",
760 "model.layers.25.mlp.gate",
761 "model.layers.25.mlp.gate.e_score_correction_bias",
762 "model.layers.25.post_attention_layernorm",
763 "model.layers.25.self_attn.A_log",
764 "model.layers.25.self_attn.b_proj",
765 "model.layers.25.self_attn.dt_bias",
766 "model.layers.25.self_attn.f_a_proj",
767 "model.layers.25.self_attn.f_b_proj",
768 "model.layers.25.self_attn.fused_qkvbfg_a_proj",
769 "model.layers.25.self_attn.g_a_proj",
770 "model.layers.25.self_attn.g_b_proj",
771 "model.layers.25.self_attn.k_conv1d",
772 "model.layers.25.self_attn.k_proj",
773 "model.layers.25.self_attn.o_norm",
774 "model.layers.25.self_attn.o_proj",
775 "model.layers.25.self_attn.q_conv1d",
776 "model.layers.25.self_attn.q_proj",
777 "model.layers.25.self_attn.qkv_proj",
778 "model.layers.25.self_attn.v_conv1d",
779 "model.layers.25.self_attn.v_proj",
780 "model.layers.26.hc_attn_base",
781 "model.layers.26.hc_attn_fn",
782 "model.layers.26.hc_attn_scale",
783 "model.layers.26.hc_ffn_base",
784 "model.layers.26.hc_ffn_fn",
785 "model.layers.26.hc_ffn_scale",
786 "model.layers.26.input_layernorm",
787 "model.layers.26.mlp.gate",
788 "model.layers.26.mlp.gate.e_score_correction_bias",
789 "model.layers.26.post_attention_layernorm",
790 "model.layers.26.self_attn.A_log",
791 "model.layers.26.self_attn.b_proj",
792 "model.layers.26.self_attn.dt_bias",
793 "model.layers.26.self_attn.f_a_proj",
794 "model.layers.26.self_attn.f_b_proj",
795 "model.layers.26.self_attn.fused_qkvbfg_a_proj",
796 "model.layers.26.self_attn.g_a_proj",
797 "model.layers.26.self_attn.g_b_proj",
798 "model.layers.26.self_attn.k_conv1d",
799 "model.layers.26.self_attn.k_proj",
800 "model.layers.26.self_attn.o_norm",
801 "model.layers.26.self_attn.o_proj",
802 "model.layers.26.self_attn.q_conv1d",
803 "model.layers.26.self_attn.q_proj",
804 "model.layers.26.self_attn.qkv_proj",
805 "model.layers.26.self_attn.v_conv1d",
806 "model.layers.26.self_attn.v_proj",
807 "model.layers.27.hc_attn_base",
808 "model.layers.27.hc_attn_fn",
809 "model.layers.27.hc_attn_scale",
810 "model.layers.27.hc_ffn_base",
811 "model.layers.27.hc_ffn_fn",
812 "model.layers.27.hc_ffn_scale",
813 "model.layers.27.input_layernorm",
814 "model.layers.27.mlp.gate",
815 "model.layers.27.mlp.gate.e_score_correction_bias",
816 "model.layers.27.post_attention_layernorm",
817 "model.layers.27.self_attn.indexer.index_kpool_compress_ape",
818 "model.layers.27.self_attn.indexer.index_kpool_compress_gate",
819 "model.layers.27.self_attn.indexer.k_norm",
820 "model.layers.27.self_attn.indexer.k_norm.bias",
821 "model.layers.27.self_attn.indexer.weights_proj",
822 "model.layers.27.self_attn.indexer.wk",
823 "model.layers.27.self_attn.indexer.wq_b",
824 "model.layers.27.self_attn.kv_a_layernorm",
825 "model.layers.27.self_attn.kv_b_proj",
826 "model.layers.27.self_attn.q_a_layernorm",
827 "model.layers.28.hc_attn_base",
828 "model.layers.28.hc_attn_fn",
829 "model.layers.28.hc_attn_scale",
830 "model.layers.28.hc_ffn_base",
831 "model.layers.28.hc_ffn_fn",
832 "model.layers.28.hc_ffn_scale",
833 "model.layers.28.input_layernorm",
834 "model.layers.28.mlp.gate",
835 "model.layers.28.mlp.gate.e_score_correction_bias",
836 "model.layers.28.post_attention_layernorm",
837 "model.layers.28.self_attn.A_log",
838 "model.layers.28.self_attn.b_proj",
839 "model.layers.28.self_attn.dt_bias",
840 "model.layers.28.self_attn.f_a_proj",
841 "model.layers.28.self_attn.f_b_proj",
842 "model.layers.28.self_attn.fused_qkvbfg_a_proj",
843 "model.layers.28.self_attn.g_a_proj",
844 "model.layers.28.self_attn.g_b_proj",
845 "model.layers.28.self_attn.k_conv1d",
846 "model.layers.28.self_attn.k_proj",
847 "model.layers.28.self_attn.o_norm",
848 "model.layers.28.self_attn.o_proj",
849 "model.layers.28.self_attn.q_conv1d",
850 "model.layers.28.self_attn.q_proj",
851 "model.layers.28.self_attn.qkv_proj",
852 "model.layers.28.self_attn.v_conv1d",
853 "model.layers.28.self_attn.v_proj",
854 "model.layers.29.hc_attn_base",
855 "model.layers.29.hc_attn_fn",
856 "model.layers.29.hc_attn_scale",
857 "model.layers.29.hc_ffn_base",
858 "model.layers.29.hc_ffn_fn",
859 "model.layers.29.hc_ffn_scale",
860 "model.layers.29.input_layernorm",
861 "model.layers.29.mlp.gate",
862 "model.layers.29.mlp.gate.e_score_correction_bias",
863 "model.layers.29.post_attention_layernorm",
864 "model.layers.29.self_attn.A_log",
865 "model.layers.29.self_attn.b_proj",
866 "model.layers.29.self_attn.dt_bias",
867 "model.layers.29.self_attn.f_a_proj",
868 "model.layers.29.self_attn.f_b_proj",
869 "model.layers.29.self_attn.fused_qkvbfg_a_proj",
870 "model.layers.29.self_attn.g_a_proj",
871 "model.layers.29.self_attn.g_b_proj",
872 "model.layers.29.self_attn.k_conv1d",
873 "model.layers.29.self_attn.k_proj",
874 "model.layers.29.self_attn.o_norm",
875 "model.layers.29.self_attn.o_proj",
876 "model.layers.29.self_attn.q_conv1d",
877 "model.layers.29.self_attn.q_proj",
878 "model.layers.29.self_attn.qkv_proj",
879 "model.layers.29.self_attn.v_conv1d",
880 "model.layers.29.self_attn.v_proj",
881 "model.layers.3.hc_attn_base",
882 "model.layers.3.hc_attn_fn",
883 "model.layers.3.hc_attn_scale",
884 "model.layers.3.hc_ffn_base",
885 "model.layers.3.hc_ffn_fn",
886 "model.layers.3.hc_ffn_scale",
887 "model.layers.3.input_layernorm",
888 "model.layers.3.mlp.gate",
889 "model.layers.3.mlp.gate.e_score_correction_bias",
890 "model.layers.3.post_attention_layernorm",
891 "model.layers.3.self_attn.indexer.index_kpool_compress_ape",
892 "model.layers.3.self_attn.indexer.index_kpool_compress_gate",
893 "model.layers.3.self_attn.indexer.k_norm",
894 "model.layers.3.self_attn.indexer.k_norm.bias",
895 "model.layers.3.self_attn.indexer.weights_proj",
896 "model.layers.3.self_attn.indexer.wk",
897 "model.layers.3.self_attn.indexer.wq_b",
898 "model.layers.3.self_attn.kv_a_layernorm",
899 "model.layers.3.self_attn.kv_b_proj",
900 "model.layers.3.self_attn.q_a_layernorm",
901 "model.layers.30.hc_attn_base",
902 "model.layers.30.hc_attn_fn",
903 "model.layers.30.hc_attn_scale",
904 "model.layers.30.hc_ffn_base",
905 "model.layers.30.hc_ffn_fn",
906 "model.layers.30.hc_ffn_scale",
907 "model.layers.30.input_layernorm",
908 "model.layers.30.mlp.gate",
909 "model.layers.30.mlp.gate.e_score_correction_bias",
910 "model.layers.30.post_attention_layernorm",
911 "model.layers.30.self_attn.A_log",
912 "model.layers.30.self_attn.b_proj",
913 "model.layers.30.self_attn.dt_bias",
914 "model.layers.30.self_attn.f_a_proj",
915 "model.layers.30.self_attn.f_b_proj",
916 "model.layers.30.self_attn.fused_qkvbfg_a_proj",
917 "model.layers.30.self_attn.g_a_proj",
918 "model.layers.30.self_attn.g_b_proj",
919 "model.layers.30.self_attn.k_conv1d",
920 "model.layers.30.self_attn.k_proj",
921 "model.layers.30.self_attn.o_norm",
922 "model.layers.30.self_attn.o_proj",
923 "model.layers.30.self_attn.q_conv1d",
924 "model.layers.30.self_attn.q_proj",
925 "model.layers.30.self_attn.qkv_proj",
926 "model.layers.30.self_attn.v_conv1d",
927 "model.layers.30.self_attn.v_proj",
928 "model.layers.31.hc_attn_base",
929 "model.layers.31.hc_attn_fn",
930 "model.layers.31.hc_attn_scale",
931 "model.layers.31.hc_ffn_base",
932 "model.layers.31.hc_ffn_fn",
933 "model.layers.31.hc_ffn_scale",
934 "model.layers.31.input_layernorm",
935 "model.layers.31.mlp.gate",
936 "model.layers.31.mlp.gate.e_score_correction_bias",
937 "model.layers.31.post_attention_layernorm",
938 "model.layers.31.self_attn.indexer.index_kpool_compress_ape",
939 "model.layers.31.self_attn.indexer.index_kpool_compress_gate",
940 "model.layers.31.self_attn.indexer.k_norm",
941 "model.layers.31.self_attn.indexer.k_norm.bias",
942 "model.layers.31.self_attn.indexer.weights_proj",
943 "model.layers.31.self_attn.indexer.wk",
944 "model.layers.31.self_attn.indexer.wq_b",
945 "model.layers.31.self_attn.kv_a_layernorm",
946 "model.layers.31.self_attn.kv_b_proj",
947 "model.layers.31.self_attn.q_a_layernorm",
948 "model.layers.32.hc_attn_base",
949 "model.layers.32.hc_attn_fn",
950 "model.layers.32.hc_attn_scale",
951 "model.layers.32.hc_ffn_base",
952 "model.layers.32.hc_ffn_fn",
953 "model.layers.32.hc_ffn_scale",
954 "model.layers.32.input_layernorm",
955 "model.layers.32.mlp.gate",
956 "model.layers.32.mlp.gate.e_score_correction_bias",
957 "model.layers.32.post_attention_layernorm",
958 "model.layers.32.self_attn.A_log",
959 "model.layers.32.self_attn.b_proj",
960 "model.layers.32.self_attn.dt_bias",
961 "model.layers.32.self_attn.f_a_proj",
962 "model.layers.32.self_attn.f_b_proj",
963 "model.layers.32.self_attn.fused_qkvbfg_a_proj",
964 "model.layers.32.self_attn.g_a_proj",
965 "model.layers.32.self_attn.g_b_proj",
966 "model.layers.32.self_attn.k_conv1d",
967 "model.layers.32.self_attn.k_proj",
968 "model.layers.32.self_attn.o_norm",
969 "model.layers.32.self_attn.o_proj",
970 "model.layers.32.self_attn.q_conv1d",
971 "model.layers.32.self_attn.q_proj",
972 "model.layers.32.self_attn.qkv_proj",
973 "model.layers.32.self_attn.v_conv1d",
974 "model.layers.32.self_attn.v_proj",
975 "model.layers.33.hc_attn_base",
976 "model.layers.33.hc_attn_fn",
977 "model.layers.33.hc_attn_scale",
978 "model.layers.33.hc_ffn_base",
979 "model.layers.33.hc_ffn_fn",
980 "model.layers.33.hc_ffn_scale",
981 "model.layers.33.input_layernorm",
982 "model.layers.33.mlp.gate",
983 "model.layers.33.mlp.gate.e_score_correction_bias",
984 "model.layers.33.post_attention_layernorm",
985 "model.layers.33.self_attn.A_log",
986 "model.layers.33.self_attn.b_proj",
987 "model.layers.33.self_attn.dt_bias",
988 "model.layers.33.self_attn.f_a_proj",
989 "model.layers.33.self_attn.f_b_proj",
990 "model.layers.33.self_attn.fused_qkvbfg_a_proj",
991 "model.layers.33.self_attn.g_a_proj",
992 "model.layers.33.self_attn.g_b_proj",
993 "model.layers.33.self_attn.k_conv1d",
994 "model.layers.33.self_attn.k_proj",
995 "model.layers.33.self_attn.o_norm",
996 "model.layers.33.self_attn.o_proj",
997 "model.layers.33.self_attn.q_conv1d",
998 "model.layers.33.self_attn.q_proj",
999 "model.layers.33.self_attn.qkv_proj",
1000 "model.layers.33.self_attn.v_conv1d",
1001 "model.layers.33.self_attn.v_proj",
1002 "model.layers.34.hc_attn_base",
1003 "model.layers.34.hc_attn_fn",
1004 "model.layers.34.hc_attn_scale",
1005 "model.layers.34.hc_ffn_base",
1006 "model.layers.34.hc_ffn_fn",
1007 "model.layers.34.hc_ffn_scale",
1008 "model.layers.34.input_layernorm",
1009 "model.layers.34.mlp.gate",
1010 "model.layers.34.mlp.gate.e_score_correction_bias",
1011 "model.layers.34.post_attention_layernorm",
1012 "model.layers.34.self_attn.A_log",
1013 "model.layers.34.self_attn.b_proj",
1014 "model.layers.34.self_attn.dt_bias",
1015 "model.layers.34.self_attn.f_a_proj",
1016 "model.layers.34.self_attn.f_b_proj",
1017 "model.layers.34.self_attn.fused_qkvbfg_a_proj",
1018 "model.layers.34.self_attn.g_a_proj",
1019 "model.layers.34.self_attn.g_b_proj",
1020 "model.layers.34.self_attn.k_conv1d",
1021 "model.layers.34.self_attn.k_proj",
1022 "model.layers.34.self_attn.o_norm",
1023 "model.layers.34.self_attn.o_proj",
1024 "model.layers.34.self_attn.q_conv1d",
1025 "model.layers.34.self_attn.q_proj",
1026 "model.layers.34.self_attn.qkv_proj",
1027 "model.layers.34.self_attn.v_conv1d",
1028 "model.layers.34.self_attn.v_proj",
1029 "model.layers.35.hc_attn_base",
1030 "model.layers.35.hc_attn_fn",
1031 "model.layers.35.hc_attn_scale",
1032 "model.layers.35.hc_ffn_base",
1033 "model.layers.35.hc_ffn_fn",
1034 "model.layers.35.hc_ffn_scale",
1035 "model.layers.35.input_layernorm",
1036 "model.layers.35.mlp.gate",
1037 "model.layers.35.mlp.gate.e_score_correction_bias",
1038 "model.layers.35.post_attention_layernorm",
1039 "model.layers.35.self_attn.indexer.index_kpool_compress_ape",
1040 "model.layers.35.self_attn.indexer.index_kpool_compress_gate",
1041 "model.layers.35.self_attn.indexer.k_norm",
1042 "model.layers.35.self_attn.indexer.k_norm.bias",
1043 "model.layers.35.self_attn.indexer.weights_proj",
1044 "model.layers.35.self_attn.indexer.wk",
1045 "model.layers.35.self_attn.indexer.wq_b",
1046 "model.layers.35.self_attn.kv_a_layernorm",
1047 "model.layers.35.self_attn.kv_b_proj",
1048 "model.layers.35.self_attn.q_a_layernorm",
1049 "model.layers.36.hc_attn_base",
1050 "model.layers.36.hc_attn_fn",
1051 "model.layers.36.hc_attn_scale",
1052 "model.layers.36.hc_ffn_base",
1053 "model.layers.36.hc_ffn_fn",
1054 "model.layers.36.hc_ffn_scale",
1055 "model.layers.36.input_layernorm",
1056 "model.layers.36.mlp.gate",
1057 "model.layers.36.mlp.gate.e_score_correction_bias",
1058 "model.layers.36.post_attention_layernorm",
1059 "model.layers.36.self_attn.A_log",
1060 "model.layers.36.self_attn.b_proj",
1061 "model.layers.36.self_attn.dt_bias",
1062 "model.layers.36.self_attn.f_a_proj",
1063 "model.layers.36.self_attn.f_b_proj",
1064 "model.layers.36.self_attn.fused_qkvbfg_a_proj",
1065 "model.layers.36.self_attn.g_a_proj",
1066 "model.layers.36.self_attn.g_b_proj",
1067 "model.layers.36.self_attn.k_conv1d",
1068 "model.layers.36.self_attn.k_proj",
1069 "model.layers.36.self_attn.o_norm",
1070 "model.layers.36.self_attn.o_proj",
1071 "model.layers.36.self_attn.q_conv1d",
1072 "model.layers.36.self_attn.q_proj",
1073 "model.layers.36.self_attn.qkv_proj",
1074 "model.layers.36.self_attn.v_conv1d",
1075 "model.layers.36.self_attn.v_proj",
1076 "model.layers.37.hc_attn_base",
1077 "model.layers.37.hc_attn_fn",
1078 "model.layers.37.hc_attn_scale",
1079 "model.layers.37.hc_ffn_base",
1080 "model.layers.37.hc_ffn_fn",
1081 "model.layers.37.hc_ffn_scale",
1082 "model.layers.37.input_layernorm",
1083 "model.layers.37.mlp.gate",
1084 "model.layers.37.mlp.gate.e_score_correction_bias",
1085 "model.layers.37.post_attention_layernorm",
1086 "model.layers.37.self_attn.A_log",
1087 "model.layers.37.self_attn.b_proj",
1088 "model.layers.37.self_attn.dt_bias",
1089 "model.layers.37.self_attn.f_a_proj",
1090 "model.layers.37.self_attn.f_b_proj",
1091 "model.layers.37.self_attn.fused_qkvbfg_a_proj",
1092 "model.layers.37.self_attn.g_a_proj",
1093 "model.layers.37.self_attn.g_b_proj",
1094 "model.layers.37.self_attn.k_conv1d",
1095 "model.layers.37.self_attn.k_proj",
1096 "model.layers.37.self_attn.o_norm",
1097 "model.layers.37.self_attn.o_proj",
1098 "model.layers.37.self_attn.q_conv1d",
1099 "model.layers.37.self_attn.q_proj",
1100 "model.layers.37.self_attn.qkv_proj",
1101 "model.layers.37.self_attn.v_conv1d",
1102 "model.layers.37.self_attn.v_proj",
1103 "model.layers.38.hc_attn_base",
1104 "model.layers.38.hc_attn_fn",
1105 "model.layers.38.hc_attn_scale",
1106 "model.layers.38.hc_ffn_base",
1107 "model.layers.38.hc_ffn_fn",
1108 "model.layers.38.hc_ffn_scale",
1109 "model.layers.38.input_layernorm",
1110 "model.layers.38.mlp.gate",
1111 "model.layers.38.mlp.gate.e_score_correction_bias",
1112 "model.layers.38.post_attention_layernorm",
1113 "model.layers.38.self_attn.A_log",
1114 "model.layers.38.self_attn.b_proj",
1115 "model.layers.38.self_attn.dt_bias",
1116 "model.layers.38.self_attn.f_a_proj",
1117 "model.layers.38.self_attn.f_b_proj",
1118 "model.layers.38.self_attn.fused_qkvbfg_a_proj",
1119 "model.layers.38.self_attn.g_a_proj",
1120 "model.layers.38.self_attn.g_b_proj",
1121 "model.layers.38.self_attn.k_conv1d",
1122 "model.layers.38.self_attn.k_proj",
1123 "model.layers.38.self_attn.o_norm",
1124 "model.layers.38.self_attn.o_proj",
1125 "model.layers.38.self_attn.q_conv1d",
1126 "model.layers.38.self_attn.q_proj",
1127 "model.layers.38.self_attn.qkv_proj",
1128 "model.layers.38.self_attn.v_conv1d",
1129 "model.layers.38.self_attn.v_proj",
1130 "model.layers.39.hc_attn_base",
1131 "model.layers.39.hc_attn_fn",
1132 "model.layers.39.hc_attn_scale",
1133 "model.layers.39.hc_ffn_base",
1134 "model.layers.39.hc_ffn_fn",
1135 "model.layers.39.hc_ffn_scale",
1136 "model.layers.39.input_layernorm",
1137 "model.layers.39.mlp.gate",
1138 "model.layers.39.mlp.gate.e_score_correction_bias",
1139 "model.layers.39.post_attention_layernorm",
1140 "model.layers.39.self_attn.indexer.index_kpool_compress_ape",
1141 "model.layers.39.self_attn.indexer.index_kpool_compress_gate",
1142 "model.layers.39.self_attn.indexer.k_norm",
1143 "model.layers.39.self_attn.indexer.k_norm.bias",
1144 "model.layers.39.self_attn.indexer.weights_proj",
1145 "model.layers.39.self_attn.indexer.wk",
1146 "model.layers.39.self_attn.indexer.wq_b",
1147 "model.layers.39.self_attn.kv_a_layernorm",
1148 "model.layers.39.self_attn.kv_b_proj",
1149 "model.layers.39.self_attn.q_a_layernorm",
1150 "model.layers.4.hc_attn_base",
1151 "model.layers.4.hc_attn_fn",
1152 "model.layers.4.hc_attn_scale",
1153 "model.layers.4.hc_ffn_base",
1154 "model.layers.4.hc_ffn_fn",
1155 "model.layers.4.hc_ffn_scale",
1156 "model.layers.4.input_layernorm",
1157 "model.layers.4.mlp.gate",
1158 "model.layers.4.mlp.gate.e_score_correction_bias",
1159 "model.layers.4.post_attention_layernorm",
1160 "model.layers.4.self_attn.A_log",
1161 "model.layers.4.self_attn.b_proj",
1162 "model.layers.4.self_attn.dt_bias",
1163 "model.layers.4.self_attn.f_a_proj",
1164 "model.layers.4.self_attn.f_b_proj",
1165 "model.layers.4.self_attn.fused_qkvbfg_a_proj",
1166 "model.layers.4.self_attn.g_a_proj",
1167 "model.layers.4.self_attn.g_b_proj",
1168 "model.layers.4.self_attn.k_conv1d",
1169 "model.layers.4.self_attn.k_proj",
1170 "model.layers.4.self_attn.o_norm",
1171 "model.layers.4.self_attn.o_proj",
1172 "model.layers.4.self_attn.q_conv1d",
1173 "model.layers.4.self_attn.q_proj",
1174 "model.layers.4.self_attn.qkv_proj",
1175 "model.layers.4.self_attn.v_conv1d",
1176 "model.layers.4.self_attn.v_proj",
1177 "model.layers.40.hc_attn_base",
1178 "model.layers.40.hc_attn_fn",
1179 "model.layers.40.hc_attn_scale",
1180 "model.layers.40.hc_ffn_base",
1181 "model.layers.40.hc_ffn_fn",
1182 "model.layers.40.hc_ffn_scale",
1183 "model.layers.40.input_layernorm",
1184 "model.layers.40.mlp.gate",
1185 "model.layers.40.mlp.gate.e_score_correction_bias",
1186 "model.layers.40.post_attention_layernorm",
1187 "model.layers.40.self_attn.A_log",
1188 "model.layers.40.self_attn.b_proj",
1189 "model.layers.40.self_attn.dt_bias",
1190 "model.layers.40.self_attn.f_a_proj",
1191 "model.layers.40.self_attn.f_b_proj",
1192 "model.layers.40.self_attn.fused_qkvbfg_a_proj",
1193 "model.layers.40.self_attn.g_a_proj",
1194 "model.layers.40.self_attn.g_b_proj",
1195 "model.layers.40.self_attn.k_conv1d",
1196 "model.layers.40.self_attn.k_proj",
1197 "model.layers.40.self_attn.o_norm",
1198 "model.layers.40.self_attn.o_proj",
1199 "model.layers.40.self_attn.q_conv1d",
1200 "model.layers.40.self_attn.q_proj",
1201 "model.layers.40.self_attn.qkv_proj",
1202 "model.layers.40.self_attn.v_conv1d",
1203 "model.layers.40.self_attn.v_proj",
1204 "model.layers.41.hc_attn_base",
1205 "model.layers.41.hc_attn_fn",
1206 "model.layers.41.hc_attn_scale",
1207 "model.layers.41.hc_ffn_base",
1208 "model.layers.41.hc_ffn_fn",
1209 "model.layers.41.hc_ffn_scale",
1210 "model.layers.41.input_layernorm",
1211 "model.layers.41.mlp.gate",
1212 "model.layers.41.mlp.gate.e_score_correction_bias",
1213 "model.layers.41.post_attention_layernorm",
1214 "model.layers.41.self_attn.A_log",
1215 "model.layers.41.self_attn.b_proj",
1216 "model.layers.41.self_attn.dt_bias",
1217 "model.layers.41.self_attn.f_a_proj",
1218 "model.layers.41.self_attn.f_b_proj",
1219 "model.layers.41.self_attn.fused_qkvbfg_a_proj",
1220 "model.layers.41.self_attn.g_a_proj",
1221 "model.layers.41.self_attn.g_b_proj",
1222 "model.layers.41.self_attn.k_conv1d",
1223 "model.layers.41.self_attn.k_proj",
1224 "model.layers.41.self_attn.o_norm",
1225 "model.layers.41.self_attn.o_proj",
1226 "model.layers.41.self_attn.q_conv1d",
1227 "model.layers.41.self_attn.q_proj",
1228 "model.layers.41.self_attn.qkv_proj",
1229 "model.layers.41.self_attn.v_conv1d",
1230 "model.layers.41.self_attn.v_proj",
1231 "model.layers.42.hc_attn_base",
1232 "model.layers.42.hc_attn_fn",
1233 "model.layers.42.hc_attn_scale",
1234 "model.layers.42.hc_ffn_base",
1235 "model.layers.42.hc_ffn_fn",
1236 "model.layers.42.hc_ffn_scale",
1237 "model.layers.42.input_layernorm",
1238 "model.layers.42.mlp.gate",
1239 "model.layers.42.mlp.gate.e_score_correction_bias",
1240 "model.layers.42.post_attention_layernorm",
1241 "model.layers.42.self_attn.A_log",
1242 "model.layers.42.self_attn.b_proj",
1243 "model.layers.42.self_attn.dt_bias",
1244 "model.layers.42.self_attn.f_a_proj",
1245 "model.layers.42.self_attn.f_b_proj",
1246 "model.layers.42.self_attn.fused_qkvbfg_a_proj",
1247 "model.layers.42.self_attn.g_a_proj",
1248 "model.layers.42.self_attn.g_b_proj",
1249 "model.layers.42.self_attn.k_conv1d",
1250 "model.layers.42.self_attn.k_proj",
1251 "model.layers.42.self_attn.o_norm",
1252 "model.layers.42.self_attn.o_proj",
1253 "model.layers.42.self_attn.q_conv1d",
1254 "model.layers.42.self_attn.q_proj",
1255 "model.layers.42.self_attn.qkv_proj",
1256 "model.layers.42.self_attn.v_conv1d",
1257 "model.layers.42.self_attn.v_proj",
1258 "model.layers.43.hc_attn_base",
1259 "model.layers.43.hc_attn_fn",
1260 "model.layers.43.hc_attn_scale",
1261 "model.layers.43.hc_ffn_base",
1262 "model.layers.43.hc_ffn_fn",
1263 "model.layers.43.hc_ffn_scale",
1264 "model.layers.43.input_layernorm",
1265 "model.layers.43.mlp.gate",
1266 "model.layers.43.mlp.gate.e_score_correction_bias",
1267 "model.layers.43.post_attention_layernorm",
1268 "model.layers.43.self_attn.indexer.index_kpool_compress_ape",
1269 "model.layers.43.self_attn.indexer.index_kpool_compress_gate",
1270 "model.layers.43.self_attn.indexer.k_norm",
1271 "model.layers.43.self_attn.indexer.k_norm.bias",
1272 "model.layers.43.self_attn.indexer.weights_proj",
1273 "model.layers.43.self_attn.indexer.wk",
1274 "model.layers.43.self_attn.indexer.wq_b",
1275 "model.layers.43.self_attn.kv_a_layernorm",
1276 "model.layers.43.self_attn.kv_b_proj",
1277 "model.layers.43.self_attn.q_a_layernorm",
1278 "model.layers.44.hc_attn_base",
1279 "model.layers.44.hc_attn_fn",
1280 "model.layers.44.hc_attn_scale",
1281 "model.layers.44.hc_ffn_base",
1282 "model.layers.44.hc_ffn_fn",
1283 "model.layers.44.hc_ffn_scale",
1284 "model.layers.44.input_layernorm",
1285 "model.layers.44.mlp.gate",
1286 "model.layers.44.mlp.gate.e_score_correction_bias",
1287 "model.layers.44.post_attention_layernorm",
1288 "model.layers.44.self_attn.A_log",
1289 "model.layers.44.self_attn.b_proj",
1290 "model.layers.44.self_attn.dt_bias",
1291 "model.layers.44.self_attn.f_a_proj",
1292 "model.layers.44.self_attn.f_b_proj",
1293 "model.layers.44.self_attn.fused_qkvbfg_a_proj",
1294 "model.layers.44.self_attn.g_a_proj",
1295 "model.layers.44.self_attn.g_b_proj",
1296 "model.layers.44.self_attn.k_conv1d",
1297 "model.layers.44.self_attn.k_proj",
1298 "model.layers.44.self_attn.o_norm",
1299 "model.layers.44.self_attn.o_proj",
1300 "model.layers.44.self_attn.q_conv1d",
1301 "model.layers.44.self_attn.q_proj",
1302 "model.layers.44.self_attn.qkv_proj",
1303 "model.layers.44.self_attn.v_conv1d",
1304 "model.layers.44.self_attn.v_proj",
1305 "model.layers.45.eh_proj",
1306 "model.layers.45.enorm",
1307 "model.layers.45.hnorm",
1308 "model.layers.45.input_layernorm",
1309 "model.layers.45.mlp.gate",
1310 "model.layers.45.mlp.gate.e_score_correction_bias",
1311 "model.layers.45.post_attention_layernorm",
1312 "model.layers.45.self_attn.indexer.index_kpool_compress_ape",
1313 "model.layers.45.self_attn.indexer.index_kpool_compress_gate",
1314 "model.layers.45.self_attn.indexer.k_norm",
1315 "model.layers.45.self_attn.indexer.k_norm.bias",
1316 "model.layers.45.self_attn.indexer.weights_proj",
1317 "model.layers.45.self_attn.indexer.wk",
1318 "model.layers.45.self_attn.indexer.wq_b",
1319 "model.layers.45.self_attn.kv_a_layernorm",
1320 "model.layers.45.self_attn.kv_b_proj",
1321 "model.layers.45.self_attn.q_a_layernorm",
1322 "model.layers.45.shared_head.norm",
1323 "model.layers.5.hc_attn_base",
1324 "model.layers.5.hc_attn_fn",
1325 "model.layers.5.hc_attn_scale",
1326 "model.layers.5.hc_ffn_base",
1327 "model.layers.5.hc_ffn_fn",
1328 "model.layers.5.hc_ffn_scale",
1329 "model.layers.5.input_layernorm",
1330 "model.layers.5.mlp.gate",
1331 "model.layers.5.mlp.gate.e_score_correction_bias",
1332 "model.layers.5.post_attention_layernorm",
1333 "model.layers.5.self_attn.A_log",
1334 "model.layers.5.self_attn.b_proj",
1335 "model.layers.5.self_attn.dt_bias",
1336 "model.layers.5.self_attn.f_a_proj",
1337 "model.layers.5.self_attn.f_b_proj",
1338 "model.layers.5.self_attn.fused_qkvbfg_a_proj",
1339 "model.layers.5.self_attn.g_a_proj",
1340 "model.layers.5.self_attn.g_b_proj",
1341 "model.layers.5.self_attn.k_conv1d",
1342 "model.layers.5.self_attn.k_proj",
1343 "model.layers.5.self_attn.o_norm",
1344 "model.layers.5.self_attn.o_proj",
1345 "model.layers.5.self_attn.q_conv1d",
1346 "model.layers.5.self_attn.q_proj",
1347 "model.layers.5.self_attn.qkv_proj",
1348 "model.layers.5.self_attn.v_conv1d",
1349 "model.layers.5.self_attn.v_proj",
1350 "model.layers.6.hc_attn_base",
1351 "model.layers.6.hc_attn_fn",
1352 "model.layers.6.hc_attn_scale",
1353 "model.layers.6.hc_ffn_base",
1354 "model.layers.6.hc_ffn_fn",
1355 "model.layers.6.hc_ffn_scale",
1356 "model.layers.6.input_layernorm",
1357 "model.layers.6.mlp.gate",
1358 "model.layers.6.mlp.gate.e_score_correction_bias",
1359 "model.layers.6.post_attention_layernorm",
1360 "model.layers.6.self_attn.A_log",
1361 "model.layers.6.self_attn.b_proj",
1362 "model.layers.6.self_attn.dt_bias",
1363 "model.layers.6.self_attn.f_a_proj",
1364 "model.layers.6.self_attn.f_b_proj",
1365 "model.layers.6.self_attn.fused_qkvbfg_a_proj",
1366 "model.layers.6.self_attn.g_a_proj",
1367 "model.layers.6.self_attn.g_b_proj",
1368 "model.layers.6.self_attn.k_conv1d",
1369 "model.layers.6.self_attn.k_proj",
1370 "model.layers.6.self_attn.o_norm",
1371 "model.layers.6.self_attn.o_proj",
1372 "model.layers.6.self_attn.q_conv1d",
1373 "model.layers.6.self_attn.q_proj",
1374 "model.layers.6.self_attn.qkv_proj",
1375 "model.layers.6.self_attn.v_conv1d",
1376 "model.layers.6.self_attn.v_proj",
1377 "model.layers.7.hc_attn_base",
1378 "model.layers.7.hc_attn_fn",
1379 "model.layers.7.hc_attn_scale",
1380 "model.layers.7.hc_ffn_base",
1381 "model.layers.7.hc_ffn_fn",
1382 "model.layers.7.hc_ffn_scale",
1383 "model.layers.7.input_layernorm",
1384 "model.layers.7.mlp.gate",
1385 "model.layers.7.mlp.gate.e_score_correction_bias",
1386 "model.layers.7.post_attention_layernorm",
1387 "model.layers.7.self_attn.indexer.index_kpool_compress_ape",
1388 "model.layers.7.self_attn.indexer.index_kpool_compress_gate",
1389 "model.layers.7.self_attn.indexer.k_norm",
1390 "model.layers.7.self_attn.indexer.k_norm.bias",
1391 "model.layers.7.self_attn.indexer.weights_proj",
1392 "model.layers.7.self_attn.indexer.wk",
1393 "model.layers.7.self_attn.indexer.wq_b",
1394 "model.layers.7.self_attn.kv_a_layernorm",
1395 "model.layers.7.self_attn.kv_b_proj",
1396 "model.layers.7.self_attn.q_a_layernorm",
1397 "model.layers.8.hc_attn_base",
1398 "model.layers.8.hc_attn_fn",
1399 "model.layers.8.hc_attn_scale",
1400 "model.layers.8.hc_ffn_base",
1401 "model.layers.8.hc_ffn_fn",
1402 "model.layers.8.hc_ffn_scale",
1403 "model.layers.8.input_layernorm",
1404 "model.layers.8.mlp.gate",
1405 "model.layers.8.mlp.gate.e_score_correction_bias",
1406 "model.layers.8.post_attention_layernorm",
1407 "model.layers.8.self_attn.A_log",
1408 "model.layers.8.self_attn.b_proj",
1409 "model.layers.8.self_attn.dt_bias",
1410 "model.layers.8.self_attn.f_a_proj",
1411 "model.layers.8.self_attn.f_b_proj",
1412 "model.layers.8.self_attn.fused_qkvbfg_a_proj",
1413 "model.layers.8.self_attn.g_a_proj",
1414 "model.layers.8.self_attn.g_b_proj",
1415 "model.layers.8.self_attn.k_conv1d",
1416 "model.layers.8.self_attn.k_proj",
1417 "model.layers.8.self_attn.o_norm",
1418 "model.layers.8.self_attn.o_proj",
1419 "model.layers.8.self_attn.q_conv1d",
1420 "model.layers.8.self_attn.q_proj",
1421 "model.layers.8.self_attn.qkv_proj",
1422 "model.layers.8.self_attn.v_conv1d",
1423 "model.layers.8.self_attn.v_proj",
1424 "model.layers.9.hc_attn_base",
1425 "model.layers.9.hc_attn_fn",
1426 "model.layers.9.hc_attn_scale",
1427 "model.layers.9.hc_ffn_base",
1428 "model.layers.9.hc_ffn_fn",
1429 "model.layers.9.hc_ffn_scale",
1430 "model.layers.9.input_layernorm",
1431 "model.layers.9.mlp.gate",
1432 "model.layers.9.mlp.gate.e_score_correction_bias",
1433 "model.layers.9.post_attention_layernorm",
1434 "model.layers.9.self_attn.A_log",
1435 "model.layers.9.self_attn.b_proj",
1436 "model.layers.9.self_attn.dt_bias",
1437 "model.layers.9.self_attn.f_a_proj",
1438 "model.layers.9.self_attn.f_b_proj",
1439 "model.layers.9.self_attn.fused_qkvbfg_a_proj",
1440 "model.layers.9.self_attn.g_a_proj",
1441 "model.layers.9.self_attn.g_b_proj",
1442 "model.layers.9.self_attn.k_conv1d",
1443 "model.layers.9.self_attn.k_proj",
1444 "model.layers.9.self_attn.o_norm",
1445 "model.layers.9.self_attn.o_proj",
1446 "model.layers.9.self_attn.q_conv1d",
1447 "model.layers.9.self_attn.q_proj",
1448 "model.layers.9.self_attn.qkv_proj",
1449 "model.layers.9.self_attn.v_conv1d",
1450 "model.layers.9.self_attn.v_proj",
1451 "model.norm",
1452 "model.visual",
1453 "router",
1454 "visual",
1455 "visual.blocks.0.attn.k_norm",
1456 "visual.blocks.0.attn.proj",
1457 "visual.blocks.0.attn.proj.bias",
1458 "visual.blocks.0.attn.q_norm",
1459 "visual.blocks.0.attn.qkv",
1460 "visual.blocks.0.attn.qkv.bias",
1461 "visual.blocks.0.mlp.down_proj",
1462 "visual.blocks.0.mlp.down_proj.bias",
1463 "visual.blocks.0.mlp.gate_proj",
1464 "visual.blocks.0.mlp.gate_proj.bias",
1465 "visual.blocks.0.mlp.up_proj",
1466 "visual.blocks.0.mlp.up_proj.bias",
1467 "visual.blocks.0.norm1",
1468 "visual.blocks.0.norm2",
1469 "visual.blocks.1.attn.k_norm",
1470 "visual.blocks.1.attn.proj",
1471 "visual.blocks.1.attn.proj.bias",
1472 "visual.blocks.1.attn.q_norm",
1473 "visual.blocks.1.attn.qkv",
1474 "visual.blocks.1.attn.qkv.bias",
1475 "visual.blocks.1.mlp.down_proj",
1476 "visual.blocks.1.mlp.down_proj.bias",
1477 "visual.blocks.1.mlp.gate_proj",
1478 "visual.blocks.1.mlp.gate_proj.bias",
1479 "visual.blocks.1.mlp.up_proj",
1480 "visual.blocks.1.mlp.up_proj.bias",
1481 "visual.blocks.1.norm1",
1482 "visual.blocks.1.norm2",
1483 "visual.blocks.10.attn.k_norm",
1484 "visual.blocks.10.attn.proj",
1485 "visual.blocks.10.attn.proj.bias",
1486 "visual.blocks.10.attn.q_norm",
1487 "visual.blocks.10.attn.qkv",
1488 "visual.blocks.10.attn.qkv.bias",
1489 "visual.blocks.10.mlp.down_proj",
1490 "visual.blocks.10.mlp.down_proj.bias",
1491 "visual.blocks.10.mlp.gate_proj",
1492 "visual.blocks.10.mlp.gate_proj.bias",
1493 "visual.blocks.10.mlp.up_proj",
1494 "visual.blocks.10.mlp.up_proj.bias",
1495 "visual.blocks.10.norm1",
1496 "visual.blocks.10.norm2",
1497 "visual.blocks.11.attn.k_norm",
1498 "visual.blocks.11.attn.proj",
1499 "visual.blocks.11.attn.proj.bias",
1500 "visual.blocks.11.attn.q_norm",
1501 "visual.blocks.11.attn.qkv",
1502 "visual.blocks.11.attn.qkv.bias",
1503 "visual.blocks.11.mlp.down_proj",
1504 "visual.blocks.11.mlp.down_proj.bias",
1505 "visual.blocks.11.mlp.gate_proj",
1506 "visual.blocks.11.mlp.gate_proj.bias",
1507 "visual.blocks.11.mlp.up_proj",
1508 "visual.blocks.11.mlp.up_proj.bias",
1509 "visual.blocks.11.norm1",
1510 "visual.blocks.11.norm2",
1511 "visual.blocks.12.attn.k_norm",
1512 "visual.blocks.12.attn.proj",
1513 "visual.blocks.12.attn.proj.bias",
1514 "visual.blocks.12.attn.q_norm",
1515 "visual.blocks.12.attn.qkv",
1516 "visual.blocks.12.attn.qkv.bias",
1517 "visual.blocks.12.mlp.down_proj",
1518 "visual.blocks.12.mlp.down_proj.bias",
1519 "visual.blocks.12.mlp.gate_proj",
1520 "visual.blocks.12.mlp.gate_proj.bias",
1521 "visual.blocks.12.mlp.up_proj",
1522 "visual.blocks.12.mlp.up_proj.bias",
1523 "visual.blocks.12.norm1",
1524 "visual.blocks.12.norm2",
1525 "visual.blocks.13.attn.k_norm",
1526 "visual.blocks.13.attn.proj",
1527 "visual.blocks.13.attn.proj.bias",
1528 "visual.blocks.13.attn.q_norm",
1529 "visual.blocks.13.attn.qkv",
1530 "visual.blocks.13.attn.qkv.bias",
1531 "visual.blocks.13.mlp.down_proj",
1532 "visual.blocks.13.mlp.down_proj.bias",
1533 "visual.blocks.13.mlp.gate_proj",
1534 "visual.blocks.13.mlp.gate_proj.bias",
1535 "visual.blocks.13.mlp.up_proj",
1536 "visual.blocks.13.mlp.up_proj.bias",
1537 "visual.blocks.13.norm1",
1538 "visual.blocks.13.norm2",
1539 "visual.blocks.14.attn.k_norm",
1540 "visual.blocks.14.attn.proj",
1541 "visual.blocks.14.attn.proj.bias",
1542 "visual.blocks.14.attn.q_norm",
1543 "visual.blocks.14.attn.qkv",
1544 "visual.blocks.14.attn.qkv.bias",
1545 "visual.blocks.14.mlp.down_proj",
1546 "visual.blocks.14.mlp.down_proj.bias",
1547 "visual.blocks.14.mlp.gate_proj",
1548 "visual.blocks.14.mlp.gate_proj.bias",
1549 "visual.blocks.14.mlp.up_proj",
1550 "visual.blocks.14.mlp.up_proj.bias",
1551 "visual.blocks.14.norm1",
1552 "visual.blocks.14.norm2",
1553 "visual.blocks.15.attn.k_norm",
1554 "visual.blocks.15.attn.proj",
1555 "visual.blocks.15.attn.proj.bias",
1556 "visual.blocks.15.attn.q_norm",
1557 "visual.blocks.15.attn.qkv",
1558 "visual.blocks.15.attn.qkv.bias",
1559 "visual.blocks.15.mlp.down_proj",
1560 "visual.blocks.15.mlp.down_proj.bias",
1561 "visual.blocks.15.mlp.gate_proj",
1562 "visual.blocks.15.mlp.gate_proj.bias",
1563 "visual.blocks.15.mlp.up_proj",
1564 "visual.blocks.15.mlp.up_proj.bias",
1565 "visual.blocks.15.norm1",
1566 "visual.blocks.15.norm2",
1567 "visual.blocks.16.attn.k_norm",
1568 "visual.blocks.16.attn.proj",
1569 "visual.blocks.16.attn.proj.bias",
1570 "visual.blocks.16.attn.q_norm",
1571 "visual.blocks.16.attn.qkv",
1572 "visual.blocks.16.attn.qkv.bias",
1573 "visual.blocks.16.mlp.down_proj",
1574 "visual.blocks.16.mlp.down_proj.bias",
1575 "visual.blocks.16.mlp.gate_proj",
1576 "visual.blocks.16.mlp.gate_proj.bias",
1577 "visual.blocks.16.mlp.up_proj",
1578 "visual.blocks.16.mlp.up_proj.bias",
1579 "visual.blocks.16.norm1",
1580 "visual.blocks.16.norm2",
1581 "visual.blocks.17.attn.k_norm",
1582 "visual.blocks.17.attn.proj",
1583 "visual.blocks.17.attn.proj.bias",
1584 "visual.blocks.17.attn.q_norm",
1585 "visual.blocks.17.attn.qkv",
1586 "visual.blocks.17.attn.qkv.bias",
1587 "visual.blocks.17.mlp.down_proj",
1588 "visual.blocks.17.mlp.down_proj.bias",
1589 "visual.blocks.17.mlp.gate_proj",
1590 "visual.blocks.17.mlp.gate_proj.bias",
1591 "visual.blocks.17.mlp.up_proj",
1592 "visual.blocks.17.mlp.up_proj.bias",
1593 "visual.blocks.17.norm1",
1594 "visual.blocks.17.norm2",
1595 "visual.blocks.18.attn.k_norm",
1596 "visual.blocks.18.attn.proj",
1597 "visual.blocks.18.attn.proj.bias",
1598 "visual.blocks.18.attn.q_norm",
1599 "visual.blocks.18.attn.qkv",
1600 "visual.blocks.18.attn.qkv.bias",
1601 "visual.blocks.18.mlp.down_proj",
1602 "visual.blocks.18.mlp.down_proj.bias",
1603 "visual.blocks.18.mlp.gate_proj",
1604 "visual.blocks.18.mlp.gate_proj.bias",
1605 "visual.blocks.18.mlp.up_proj",
1606 "visual.blocks.18.mlp.up_proj.bias",
1607 "visual.blocks.18.norm1",
1608 "visual.blocks.18.norm2",
1609 "visual.blocks.19.attn.k_norm",
1610 "visual.blocks.19.attn.proj",
1611 "visual.blocks.19.attn.proj.bias",
1612 "visual.blocks.19.attn.q_norm",
1613 "visual.blocks.19.attn.qkv",
1614 "visual.blocks.19.attn.qkv.bias",
1615 "visual.blocks.19.mlp.down_proj",
1616 "visual.blocks.19.mlp.down_proj.bias",
1617 "visual.blocks.19.mlp.gate_proj",
1618 "visual.blocks.19.mlp.gate_proj.bias",
1619 "visual.blocks.19.mlp.up_proj",
1620 "visual.blocks.19.mlp.up_proj.bias",
1621 "visual.blocks.19.norm1",
1622 "visual.blocks.19.norm2",
1623 "visual.blocks.2.attn.k_norm",
1624 "visual.blocks.2.attn.proj",
1625 "visual.blocks.2.attn.proj.bias",
1626 "visual.blocks.2.attn.q_norm",
1627 "visual.blocks.2.attn.qkv",
1628 "visual.blocks.2.attn.qkv.bias",
1629 "visual.blocks.2.mlp.down_proj",
1630 "visual.blocks.2.mlp.down_proj.bias",
1631 "visual.blocks.2.mlp.gate_proj",
1632 "visual.blocks.2.mlp.gate_proj.bias",
1633 "visual.blocks.2.mlp.up_proj",
1634 "visual.blocks.2.mlp.up_proj.bias",
1635 "visual.blocks.2.norm1",
1636 "visual.blocks.2.norm2",
1637 "visual.blocks.20.attn.k_norm",
1638 "visual.blocks.20.attn.proj",
1639 "visual.blocks.20.attn.proj.bias",
1640 "visual.blocks.20.attn.q_norm",
1641 "visual.blocks.20.attn.qkv",
1642 "visual.blocks.20.attn.qkv.bias",
1643 "visual.blocks.20.mlp.down_proj",
1644 "visual.blocks.20.mlp.down_proj.bias",
1645 "visual.blocks.20.mlp.gate_proj",
1646 "visual.blocks.20.mlp.gate_proj.bias",
1647 "visual.blocks.20.mlp.up_proj",
1648 "visual.blocks.20.mlp.up_proj.bias",
1649 "visual.blocks.20.norm1",
1650 "visual.blocks.20.norm2",
1651 "visual.blocks.21.attn.k_norm",
1652 "visual.blocks.21.attn.proj",
1653 "visual.blocks.21.attn.proj.bias",
1654 "visual.blocks.21.attn.q_norm",
1655 "visual.blocks.21.attn.qkv",
1656 "visual.blocks.21.attn.qkv.bias",
1657 "visual.blocks.21.mlp.down_proj",
1658 "visual.blocks.21.mlp.down_proj.bias",
1659 "visual.blocks.21.mlp.gate_proj",
1660 "visual.blocks.21.mlp.gate_proj.bias",
1661 "visual.blocks.21.mlp.up_proj",
1662 "visual.blocks.21.mlp.up_proj.bias",
1663 "visual.blocks.21.norm1",
1664 "visual.blocks.21.norm2",
1665 "visual.blocks.22.attn.k_norm",
1666 "visual.blocks.22.attn.proj",
1667 "visual.blocks.22.attn.proj.bias",
1668 "visual.blocks.22.attn.q_norm",
1669 "visual.blocks.22.attn.qkv",
1670 "visual.blocks.22.attn.qkv.bias",
1671 "visual.blocks.22.mlp.down_proj",
1672 "visual.blocks.22.mlp.down_proj.bias",
1673 "visual.blocks.22.mlp.gate_proj",
1674 "visual.blocks.22.mlp.gate_proj.bias",
1675 "visual.blocks.22.mlp.up_proj",
1676 "visual.blocks.22.mlp.up_proj.bias",
1677 "visual.blocks.22.norm1",
1678 "visual.blocks.22.norm2",
1679 "visual.blocks.23.attn.k_norm",
1680 "visual.blocks.23.attn.proj",
1681 "visual.blocks.23.attn.proj.bias",
1682 "visual.blocks.23.attn.q_norm",
1683 "visual.blocks.23.attn.qkv",
1684 "visual.blocks.23.attn.qkv.bias",
1685 "visual.blocks.23.mlp.down_proj",
1686 "visual.blocks.23.mlp.down_proj.bias",
1687 "visual.blocks.23.mlp.gate_proj",
1688 "visual.blocks.23.mlp.gate_proj.bias",
1689 "visual.blocks.23.mlp.up_proj",
1690 "visual.blocks.23.mlp.up_proj.bias",
1691 "visual.blocks.23.norm1",
1692 "visual.blocks.23.norm2",
1693 "visual.blocks.3.attn.k_norm",
1694 "visual.blocks.3.attn.proj",
1695 "visual.blocks.3.attn.proj.bias",
1696 "visual.blocks.3.attn.q_norm",
1697 "visual.blocks.3.attn.qkv",
1698 "visual.blocks.3.attn.qkv.bias",
1699 "visual.blocks.3.mlp.down_proj",
1700 "visual.blocks.3.mlp.down_proj.bias",
1701 "visual.blocks.3.mlp.gate_proj",
1702 "visual.blocks.3.mlp.gate_proj.bias",
1703 "visual.blocks.3.mlp.up_proj",
1704 "visual.blocks.3.mlp.up_proj.bias",
1705 "visual.blocks.3.norm1",
1706 "visual.blocks.3.norm2",
1707 "visual.blocks.4.attn.k_norm",
1708 "visual.blocks.4.attn.proj",
1709 "visual.blocks.4.attn.proj.bias",
1710 "visual.blocks.4.attn.q_norm",
1711 "visual.blocks.4.attn.qkv",
1712 "visual.blocks.4.attn.qkv.bias",
1713 "visual.blocks.4.mlp.down_proj",
1714 "visual.blocks.4.mlp.down_proj.bias",
1715 "visual.blocks.4.mlp.gate_proj",
1716 "visual.blocks.4.mlp.gate_proj.bias",
1717 "visual.blocks.4.mlp.up_proj",
1718 "visual.blocks.4.mlp.up_proj.bias",
1719 "visual.blocks.4.norm1",
1720 "visual.blocks.4.norm2",
1721 "visual.blocks.5.attn.k_norm",
1722 "visual.blocks.5.attn.proj",
1723 "visual.blocks.5.attn.proj.bias",
1724 "visual.blocks.5.attn.q_norm",
1725 "visual.blocks.5.attn.qkv",
1726 "visual.blocks.5.attn.qkv.bias",
1727 "visual.blocks.5.mlp.down_proj",
1728 "visual.blocks.5.mlp.down_proj.bias",
1729 "visual.blocks.5.mlp.gate_proj",
1730 "visual.blocks.5.mlp.gate_proj.bias",
1731 "visual.blocks.5.mlp.up_proj",
1732 "visual.blocks.5.mlp.up_proj.bias",
1733 "visual.blocks.5.norm1",
1734 "visual.blocks.5.norm2",
1735 "visual.blocks.6.attn.k_norm",
1736 "visual.blocks.6.attn.proj",
1737 "visual.blocks.6.attn.proj.bias",
1738 "visual.blocks.6.attn.q_norm",
1739 "visual.blocks.6.attn.qkv",
1740 "visual.blocks.6.attn.qkv.bias",
1741 "visual.blocks.6.mlp.down_proj",
1742 "visual.blocks.6.mlp.down_proj.bias",
1743 "visual.blocks.6.mlp.gate_proj",
1744 "visual.blocks.6.mlp.gate_proj.bias",
1745 "visual.blocks.6.mlp.up_proj",
1746 "visual.blocks.6.mlp.up_proj.bias",
1747 "visual.blocks.6.norm1",
1748 "visual.blocks.6.norm2",
1749 "visual.blocks.7.attn.k_norm",
1750 "visual.blocks.7.attn.proj",
1751 "visual.blocks.7.attn.proj.bias",
1752 "visual.blocks.7.attn.q_norm",
1753 "visual.blocks.7.attn.qkv",
1754 "visual.blocks.7.attn.qkv.bias",
1755 "visual.blocks.7.mlp.down_proj",
1756 "visual.blocks.7.mlp.down_proj.bias",
1757 "visual.blocks.7.mlp.gate_proj",
1758 "visual.blocks.7.mlp.gate_proj.bias",
1759 "visual.blocks.7.mlp.up_proj",
1760 "visual.blocks.7.mlp.up_proj.bias",
1761 "visual.blocks.7.norm1",
1762 "visual.blocks.7.norm2",
1763 "visual.blocks.8.attn.k_norm",
1764 "visual.blocks.8.attn.proj",
1765 "visual.blocks.8.attn.proj.bias",
1766 "visual.blocks.8.attn.q_norm",
1767 "visual.blocks.8.attn.qkv",
1768 "visual.blocks.8.attn.qkv.bias",
1769 "visual.blocks.8.mlp.down_proj",
1770 "visual.blocks.8.mlp.down_proj.bias",
1771 "visual.blocks.8.mlp.gate_proj",
1772 "visual.blocks.8.mlp.gate_proj.bias",
1773 "visual.blocks.8.mlp.up_proj",
1774 "visual.blocks.8.mlp.up_proj.bias",
1775 "visual.blocks.8.norm1",
1776 "visual.blocks.8.norm2",
1777 "visual.blocks.9.attn.k_norm",
1778 "visual.blocks.9.attn.proj",
1779 "visual.blocks.9.attn.proj.bias",
1780 "visual.blocks.9.attn.q_norm",
1781 "visual.blocks.9.attn.qkv",
1782 "visual.blocks.9.attn.qkv.bias",
1783 "visual.blocks.9.mlp.down_proj",
1784 "visual.blocks.9.mlp.down_proj.bias",
1785 "visual.blocks.9.mlp.gate_proj",
1786 "visual.blocks.9.mlp.gate_proj.bias",
1787 "visual.blocks.9.mlp.up_proj",
1788 "visual.blocks.9.mlp.up_proj.bias",
1789 "visual.blocks.9.norm1",
1790 "visual.blocks.9.norm2",
1791 "visual.downsample",
1792 "visual.downsample.bias",
1793 "visual.merger.down_proj",
1794 "visual.merger.gate_proj",
1795 "visual.merger.post_projection_norm",
1796 "visual.merger.post_projection_norm.bias",
1797 "visual.merger.proj",
1798 "visual.merger.up_proj",
1799 "visual.patch_embed.proj",
1800 "visual.patch_embed.proj.bias",
1801 "visual.post_layernorm",
1802 "weights_proj"
1803 ],
1804 "quant_method": "fp8",
1805 "weight_block_size": [
1806 128,
1807 128
1808 ]
1809 }
1810 }
1811