config.json
481 B · 21 lines · json Raw
1 {
2 "architectures": [
3 "Qwen2ForCausalLM"
4 ],
5 "model_type": "qwen2",
6 "vocab_size": 151936,
7 "hidden_size": 1536,
8 "intermediate_size": 8960,
9 "num_hidden_layers": 28,
10 "num_attention_heads": 12,
11 "num_key_value_heads": 2,
12 "hidden_act": "silu",
13 "max_position_embeddings": 131072,
14 "rms_norm_eps": 1e-06,
15 "rope_theta": 10000.0,
16 "tie_word_embeddings": false,
17 "torch_dtype": "bfloat16",
18 "use_cache": true,
19 "bos_token_id": 151646,
20 "eos_token_id": 151643
21 }