config.json
4.3 KB · 173 lines · json Raw
1 {
2 "architectures": [
3 "Gemma4UnifiedForConditionalGeneration"
4 ],
5 "audio_config": {
6 "_name_or_path": "",
7 "architectures": null,
8 "audio_embed_dim": 640,
9 "audio_samples_per_token": 640,
10 "chunk_size_feed_forward": 0,
11 "dtype": null,
12 "hidden_size": 640,
13 "id2label": {
14 "0": "LABEL_0",
15 "1": "LABEL_1"
16 },
17 "initializer_range": 0.02,
18 "is_encoder_decoder": false,
19 "label2id": {
20 "LABEL_0": 0,
21 "LABEL_1": 1
22 },
23 "model_type": "gemma4_unified_audio",
24 "output_attentions": false,
25 "output_hidden_states": false,
26 "output_proj_dims": 640,
27 "problem_type": null,
28 "return_dict": true,
29 "rms_norm_eps": 1e-06
30 },
31 "audio_token_id": 258881,
32 "boa_token_id": 256000,
33 "boi_token_id": 255999,
34 "dtype": "bfloat16",
35 "eoa_token_index": 258883,
36 "eoi_token_id": 258882,
37 "eos_token_id": [
38 1,
39 106
40 ],
41 "image_token_id": 258880,
42 "initializer_range": 0.02,
43 "model_type": "gemma4_unified",
44 "text_config": {
45 "attention_bias": false,
46 "attention_dropout": 0.0,
47 "attention_k_eq_v": true,
48 "bos_token_id": 2,
49 "enable_moe_block": false,
50 "eos_token_id": 1,
51 "final_logit_softcapping": 30.0,
52 "global_head_dim": 512,
53 "head_dim": 256,
54 "hidden_activation": "gelu_pytorch_tanh",
55 "hidden_size": 3840,
56 "hidden_size_per_layer_input": 0,
57 "initializer_range": 0.02,
58 "intermediate_size": 15360,
59 "layer_types": [
60 "sliding_attention",
61 "sliding_attention",
62 "sliding_attention",
63 "sliding_attention",
64 "sliding_attention",
65 "full_attention",
66 "sliding_attention",
67 "sliding_attention",
68 "sliding_attention",
69 "sliding_attention",
70 "sliding_attention",
71 "full_attention",
72 "sliding_attention",
73 "sliding_attention",
74 "sliding_attention",
75 "sliding_attention",
76 "sliding_attention",
77 "full_attention",
78 "sliding_attention",
79 "sliding_attention",
80 "sliding_attention",
81 "sliding_attention",
82 "sliding_attention",
83 "full_attention",
84 "sliding_attention",
85 "sliding_attention",
86 "sliding_attention",
87 "sliding_attention",
88 "sliding_attention",
89 "full_attention",
90 "sliding_attention",
91 "sliding_attention",
92 "sliding_attention",
93 "sliding_attention",
94 "sliding_attention",
95 "full_attention",
96 "sliding_attention",
97 "sliding_attention",
98 "sliding_attention",
99 "sliding_attention",
100 "sliding_attention",
101 "full_attention",
102 "sliding_attention",
103 "sliding_attention",
104 "sliding_attention",
105 "sliding_attention",
106 "sliding_attention",
107 "full_attention"
108 ],
109 "max_position_embeddings": 262144,
110 "model_type": "gemma4_unified_text",
111 "moe_intermediate_size": null,
112 "num_attention_heads": 16,
113 "num_experts": null,
114 "num_global_key_value_heads": 1,
115 "num_hidden_layers": 48,
116 "num_key_value_heads": 8,
117 "num_kv_shared_layers": 0,
118 "pad_token_id": 0,
119 "rms_norm_eps": 1e-06,
120 "rope_parameters": {
121 "full_attention": {
122 "partial_rotary_factor": 0.25,
123 "rope_theta": 1000000.0,
124 "rope_type": "proportional"
125 },
126 "sliding_attention": {
127 "rope_theta": 10000.0,
128 "rope_type": "default"
129 }
130 },
131 "sliding_window": 1024,
132 "tie_word_embeddings": true,
133 "top_k_experts": null,
134 "use_bidirectional_attention": "vision",
135 "use_cache": true,
136 "use_double_wide_mlp": false,
137 "vocab_size": 262144,
138 "vocab_size_per_layer_input": 262144
139 },
140 "tie_word_embeddings": true,
141 "transformers_version": "5.10.0.dev0",
142 "video_token_id": 258884,
143 "vision_config": {
144 "_name_or_path": "",
145 "architectures": null,
146 "chunk_size_feed_forward": 0,
147 "dtype": null,
148 "id2label": {
149 "0": "LABEL_0",
150 "1": "LABEL_1"
151 },
152 "initializer_range": 0.02,
153 "is_encoder_decoder": false,
154 "label2id": {
155 "LABEL_0": 0,
156 "LABEL_1": 1
157 },
158 "mm_embed_dim": 3840,
159 "mm_posemb_size": 1120,
160 "model_patch_size": 48,
161 "model_type": "gemma4_unified_vision",
162 "num_soft_tokens": 280,
163 "output_attentions": false,
164 "output_hidden_states": false,
165 "output_proj_dims": 3840,
166 "patch_size": 16,
167 "pooling_kernel_size": 3,
168 "problem_type": null,
169 "return_dict": true,
170 "rms_norm_eps": 1e-06
171 }
172 }
173