config.json
2.7 KB · 112 lines · json Raw
1 {
2 "attention_bias": false,
3 "attention_dropout": 0.0,
4 "attention_k_eq_v": true,
5 "bos_token_id": 2,
6 "dtype": "bfloat16",
7 "enable_moe_block": false,
8 "eos_token_id": 106,
9 "expert_intermediate_size": null,
10 "final_logit_softcapping": 30.0,
11 "global_head_dim": 512,
12 "head_dim": 256,
13 "hidden_activation": "gelu_pytorch_tanh",
14 "hidden_size": 5376,
15 "hidden_size_per_layer_input": 0,
16 "initializer_range": 0.02,
17 "intermediate_size": 21504,
18 "layer_types": [
19 "sliding_attention",
20 "sliding_attention",
21 "sliding_attention",
22 "sliding_attention",
23 "sliding_attention",
24 "full_attention",
25 "sliding_attention",
26 "sliding_attention",
27 "sliding_attention",
28 "sliding_attention",
29 "sliding_attention",
30 "full_attention",
31 "sliding_attention",
32 "sliding_attention",
33 "sliding_attention",
34 "sliding_attention",
35 "sliding_attention",
36 "full_attention",
37 "sliding_attention",
38 "sliding_attention",
39 "sliding_attention",
40 "sliding_attention",
41 "sliding_attention",
42 "full_attention",
43 "sliding_attention",
44 "sliding_attention",
45 "sliding_attention",
46 "sliding_attention",
47 "sliding_attention",
48 "full_attention",
49 "sliding_attention",
50 "sliding_attention",
51 "sliding_attention",
52 "sliding_attention",
53 "sliding_attention",
54 "full_attention",
55 "sliding_attention",
56 "sliding_attention",
57 "sliding_attention",
58 "sliding_attention",
59 "sliding_attention",
60 "full_attention",
61 "sliding_attention",
62 "sliding_attention",
63 "sliding_attention",
64 "sliding_attention",
65 "sliding_attention",
66 "full_attention",
67 "sliding_attention",
68 "sliding_attention",
69 "sliding_attention",
70 "sliding_attention",
71 "sliding_attention",
72 "full_attention",
73 "sliding_attention",
74 "sliding_attention",
75 "sliding_attention",
76 "sliding_attention",
77 "sliding_attention",
78 "full_attention"
79 ],
80 "max_position_embeddings": 262144,
81 "model_type": "gemma4_text",
82 "moe_intermediate_size": null,
83 "num_attention_heads": 32,
84 "num_experts": null,
85 "num_global_key_value_heads": 4,
86 "num_hidden_layers": 60,
87 "num_key_value_heads": 16,
88 "num_kv_shared_layers": 0,
89 "pad_token_id": 0,
90 "rms_norm_eps": 1e-06,
91 "rope_parameters": {
92 "full_attention": {
93 "partial_rotary_factor": 0.25,
94 "rope_theta": 1000000.0,
95 "rope_type": "proportional"
96 },
97 "sliding_attention": {
98 "rope_theta": 10000.0,
99 "rope_type": "default"
100 }
101 },
102 "sliding_window": 1024,
103 "tie_word_embeddings": true,
104 "top_k_experts": null,
105 "transformers_version": "5.5.0",
106 "use_bidirectional_attention": "vision",
107 "use_cache": false,
108 "use_double_wide_mlp": false,
109 "vocab_size": 262144,
110 "vocab_size_per_layer_input": 262144
111 }
112