config.json
1.9 KB · 114 lines · json Raw
1 {
2 "architectures": [
3 "SmolLM3ForCausalLM"
4 ],
5 "attention_bias": false,
6 "attention_dropout": 0.0,
7 "bos_token_id": 128000,
8 "dtype": "bfloat16",
9 "eos_token_id": 128012,
10 "hidden_act": "silu",
11 "hidden_size": 2048,
12 "initializer_range": 0.02,
13 "intermediate_size": 11008,
14 "layer_types": [
15 "full_attention",
16 "full_attention",
17 "full_attention",
18 "full_attention",
19 "full_attention",
20 "full_attention",
21 "full_attention",
22 "full_attention",
23 "full_attention",
24 "full_attention",
25 "full_attention",
26 "full_attention",
27 "full_attention",
28 "full_attention",
29 "full_attention",
30 "full_attention",
31 "full_attention",
32 "full_attention",
33 "full_attention",
34 "full_attention",
35 "full_attention",
36 "full_attention",
37 "full_attention",
38 "full_attention",
39 "full_attention",
40 "full_attention",
41 "full_attention",
42 "full_attention",
43 "full_attention",
44 "full_attention",
45 "full_attention",
46 "full_attention",
47 "full_attention",
48 "full_attention",
49 "full_attention",
50 "full_attention"
51 ],
52 "max_position_embeddings": 65536,
53 "max_window_layers": 28,
54 "mlp_bias": false,
55 "model_type": "smollm3",
56 "no_rope_layer_interval": 4,
57 "no_rope_layers": [
58 1,
59 1,
60 1,
61 0,
62 1,
63 1,
64 1,
65 0,
66 1,
67 1,
68 1,
69 0,
70 1,
71 1,
72 1,
73 0,
74 1,
75 1,
76 1,
77 0,
78 1,
79 1,
80 1,
81 0,
82 1,
83 1,
84 1,
85 0,
86 1,
87 1,
88 1,
89 0,
90 1,
91 1,
92 1,
93 0
94 ],
95 "num_attention_heads": 16,
96 "num_hidden_layers": 36,
97 "num_key_value_heads": 4,
98 "pad_token_id": 128004,
99 "pretraining_tp": 1,
100 "rms_norm_eps": 1e-06,
101 "rope_parameters": {
102 "rope_theta": 5000000.0,
103 "rope_type": "default"
104 },
105 "sliding_window": null,
106 "tie_word_embeddings": true,
107 "transformers_version": "5.5.0",
108 "use_cache": false,
109 "use_sliding_window": false,
110 "vocab_size": 128256,
111 "rope_theta": 5000000.0,
112 "torch_dtype": "bfloat16"
113 }
114