config.json
4.1 KB · 157 lines · json Raw
1 {
2 "architectures": [
3 "VLXSeek1_5ForCausalLM"
4 ],
5 "aux_image_aspect_ratio": "dynamic",
6 "aux_image_size": 1024,
7 "delay_load": true,
8 "dtype": "bfloat16",
9 "image_token_id": 248056,
10 "mm_add_pos_embed": true,
11 "mm_apply_object_layer_norm": false,
12 "mm_bbox_order_mode": "raster",
13 "mm_hidden_size": 4096,
14 "mm_num_object_tokens": 100,
15 "mm_obj_pooling_type": "mean",
16 "mm_object_feature_combination": "concat",
17 "mm_object_hidden_size": 6656,
18 "mm_projector_aux_type": "mlp2x_gelu",
19 "mm_projector_type": "identity",
20 "mm_roi_algined": true,
21 "mm_roi_output_size": 14,
22 "mm_simpleFPN_out_channels_for_vt": 512,
23 "mm_use_multi_scale_roi_align": false,
24 "mm_use_separate_mlp_for_object": false,
25 "mm_use_simpleFPN_for_vt": true,
26 "mm_use_simpleFPN_for_vt_aux": false,
27 "mm_use_vision_tower_object_feature": true,
28 "mm_use_vt_object_feature_only": false,
29 "mm_vision_tower": "resources/Qwen3.5-9B-Vision_Tower",
30 "mm_vision_tower_aux": "c-radio_v4-so400m",
31 "model_type": "vlx_seek_1_5",
32 "pad_token_id": 248044,
33 "text_config": {
34 "attention_bias": false,
35 "attention_dropout": 0.0,
36 "attn_output_gate": true,
37 "bos_token_id": null,
38 "dtype": "bfloat16",
39 "eos_token_id": 248044,
40 "full_attention_interval": 4,
41 "head_dim": 256,
42 "hidden_act": "silu",
43 "hidden_size": 4096,
44 "initializer_range": 0.02,
45 "intermediate_size": 12288,
46 "layer_types": [
47 "linear_attention",
48 "linear_attention",
49 "linear_attention",
50 "full_attention",
51 "linear_attention",
52 "linear_attention",
53 "linear_attention",
54 "full_attention",
55 "linear_attention",
56 "linear_attention",
57 "linear_attention",
58 "full_attention",
59 "linear_attention",
60 "linear_attention",
61 "linear_attention",
62 "full_attention",
63 "linear_attention",
64 "linear_attention",
65 "linear_attention",
66 "full_attention",
67 "linear_attention",
68 "linear_attention",
69 "linear_attention",
70 "full_attention",
71 "linear_attention",
72 "linear_attention",
73 "linear_attention",
74 "full_attention",
75 "linear_attention",
76 "linear_attention",
77 "linear_attention",
78 "full_attention"
79 ],
80 "linear_conv_kernel_dim": 4,
81 "linear_key_head_dim": 128,
82 "linear_num_key_heads": 16,
83 "linear_num_value_heads": 32,
84 "linear_value_head_dim": 128,
85 "mamba_ssm_dtype": "float32",
86 "max_position_embeddings": 262144,
87 "mlp_only_layers": [],
88 "model_type": "qwen3_5_text",
89 "mtp_num_hidden_layers": 1,
90 "mtp_use_dedicated_embeddings": false,
91 "num_attention_heads": 16,
92 "num_hidden_layers": 32,
93 "num_key_value_heads": 4,
94 "pad_token_id": null,
95 "partial_rotary_factor": 0.25,
96 "rms_norm_eps": 1e-06,
97 "rope_parameters": {
98 "mrope_interleaved": true,
99 "mrope_section": [
100 11,
101 11,
102 10
103 ],
104 "partial_rotary_factor": 0.25,
105 "rope_theta": 10000000,
106 "rope_type": "default"
107 },
108 "tie_word_embeddings": false,
109 "use_cache": true,
110 "vocab_size": 248182
111 },
112 "tie_word_embeddings": false,
113 "transformers_version": "5.2.0",
114 "use_cache": true,
115 "video_token_id": 248057,
116 "vision_aux_config": {
117 "args": {
118 "cpe_max_size": 2048,
119 "cpe_num_registers": null,
120 "dtype": "float32",
121 "model": "vit_so400m_patch16_224",
122 "num_cls_tokens": 3,
123 "register_multiple": 10
124 },
125 "max_resolution": 2048,
126 "patch_size": 16,
127 "preferred_resolution": [
128 512,
129 512
130 ],
131 "summary_idxs": [
132 0,
133 1
134 ],
135 "version": "c-radio_v4-so400m"
136 },
137 "vision_config": {
138 "deepstack_visual_indexes": [],
139 "depth": 27,
140 "dtype": "bfloat16",
141 "hidden_act": "gelu_pytorch_tanh",
142 "hidden_size": 1152,
143 "in_channels": 3,
144 "initializer_range": 0.02,
145 "intermediate_size": 4304,
146 "model_type": "qwen3_5",
147 "num_heads": 16,
148 "num_position_embeddings": 2304,
149 "out_hidden_size": 4096,
150 "patch_size": 16,
151 "spatial_merge_size": 2,
152 "temporal_patch_size": 2
153 },
154 "vision_end_token_id": 248054,
155 "vision_start_token_id": 248053
156 }
157