config.json
4.5 KB · 177 lines · json Raw
1 {
2 "architectures": [
3 "Gemma4ForConditionalGeneration"
4 ],
5 "audio_config": null,
6 "audio_token_id": 258881,
7 "boa_token_id": 256000,
8 "boi_token_id": 255999,
9 "dtype": "bfloat16",
10 "eoa_token_id": 258883,
11 "eoa_token_index": 258883,
12 "eoi_token_id": 258882,
13 "eos_token_id": [
14 1,
15 106
16 ],
17 "image_token_id": 258880,
18 "initializer_range": 0.02,
19 "model_type": "gemma4",
20 "text_config": {
21 "attention_bias": false,
22 "attention_dropout": 0.0,
23 "attention_k_eq_v": true,
24 "bos_token_id": 2,
25 "dtype": "bfloat16",
26 "enable_moe_block": false,
27 "eos_token_id": 1,
28 "expert_intermediate_size": null,
29 "final_logit_softcapping": 30.0,
30 "global_head_dim": 512,
31 "head_dim": 256,
32 "hidden_activation": "gelu_pytorch_tanh",
33 "hidden_size": 5376,
34 "hidden_size_per_layer_input": 0,
35 "initializer_range": 0.02,
36 "intermediate_size": 21504,
37 "layer_types": [
38 "sliding_attention",
39 "sliding_attention",
40 "sliding_attention",
41 "sliding_attention",
42 "sliding_attention",
43 "full_attention",
44 "sliding_attention",
45 "sliding_attention",
46 "sliding_attention",
47 "sliding_attention",
48 "sliding_attention",
49 "full_attention",
50 "sliding_attention",
51 "sliding_attention",
52 "sliding_attention",
53 "sliding_attention",
54 "sliding_attention",
55 "full_attention",
56 "sliding_attention",
57 "sliding_attention",
58 "sliding_attention",
59 "sliding_attention",
60 "sliding_attention",
61 "full_attention",
62 "sliding_attention",
63 "sliding_attention",
64 "sliding_attention",
65 "sliding_attention",
66 "sliding_attention",
67 "full_attention",
68 "sliding_attention",
69 "sliding_attention",
70 "sliding_attention",
71 "sliding_attention",
72 "sliding_attention",
73 "full_attention",
74 "sliding_attention",
75 "sliding_attention",
76 "sliding_attention",
77 "sliding_attention",
78 "sliding_attention",
79 "full_attention",
80 "sliding_attention",
81 "sliding_attention",
82 "sliding_attention",
83 "sliding_attention",
84 "sliding_attention",
85 "full_attention",
86 "sliding_attention",
87 "sliding_attention",
88 "sliding_attention",
89 "sliding_attention",
90 "sliding_attention",
91 "full_attention",
92 "sliding_attention",
93 "sliding_attention",
94 "sliding_attention",
95 "sliding_attention",
96 "sliding_attention",
97 "full_attention"
98 ],
99 "max_position_embeddings": 262144,
100 "model_type": "gemma4_text",
101 "num_attention_heads": 32,
102 "num_experts": null,
103 "num_global_key_value_heads": 4,
104 "num_hidden_layers": 60,
105 "num_key_value_heads": 16,
106 "num_kv_shared_layers": 0,
107 "pad_token_id": 0,
108 "rms_norm_eps": 1e-06,
109 "rope_parameters": {
110 "full_attention": {
111 "partial_rotary_factor": 0.25,
112 "rope_theta": 1000000.0,
113 "rope_type": "proportional"
114 },
115 "sliding_attention": {
116 "rope_theta": 10000.0,
117 "rope_type": "default"
118 }
119 },
120 "sliding_window": 1024,
121 "tie_word_embeddings": true,
122 "top_k_experts": null,
123 "use_bidirectional_attention": "vision",
124 "use_cache": true,
125 "use_double_wide_mlp": false,
126 "vocab_size": 262144,
127 "vocab_size_per_layer_input": 262144
128 },
129 "tie_word_embeddings": true,
130 "transformers_version": "5.5.0.dev0",
131 "video_token_id": 258884,
132 "vision_config": {
133 "_name_or_path": "",
134 "architectures": null,
135 "attention_bias": false,
136 "attention_dropout": 0.0,
137 "chunk_size_feed_forward": 0,
138 "default_output_length": 280,
139 "dtype": "bfloat16",
140 "global_head_dim": 72,
141 "head_dim": 72,
142 "hidden_activation": "gelu_pytorch_tanh",
143 "hidden_size": 1152,
144 "id2label": {
145 "0": "LABEL_0",
146 "1": "LABEL_1"
147 },
148 "initializer_range": 0.02,
149 "intermediate_size": 4304,
150 "is_encoder_decoder": false,
151 "label2id": {
152 "LABEL_0": 0,
153 "LABEL_1": 1
154 },
155 "max_position_embeddings": 131072,
156 "model_type": "gemma4_vision",
157 "num_attention_heads": 16,
158 "num_hidden_layers": 27,
159 "num_key_value_heads": 16,
160 "output_attentions": false,
161 "output_hidden_states": false,
162 "patch_size": 16,
163 "pooling_kernel_size": 3,
164 "position_embedding_size": 10240,
165 "problem_type": null,
166 "return_dict": true,
167 "rms_norm_eps": 1e-06,
168 "rope_parameters": {
169 "rope_theta": 100.0,
170 "rope_type": "default"
171 },
172 "standardize": true,
173 "use_clipped_linears": false
174 },
175 "vision_soft_tokens_per_image": 280
176 }
177