config.json
484 B · 21 lines · json Raw
1 {
2 "architectures": [
3 "Qwen2ForCausalLM"
4 ],
5 "model_type": "qwen2",
6 "vocab_size": 152064,
7 "hidden_size": 5120,
8 "intermediate_size": 13824,
9 "num_hidden_layers": 48,
10 "num_attention_heads": 40,
11 "num_key_value_heads": 8,
12 "hidden_act": "silu",
13 "max_position_embeddings": 131072,
14 "rms_norm_eps": 1e-05,
15 "rope_theta": 1000000.0,
16 "tie_word_embeddings": false,
17 "torch_dtype": "bfloat16",
18 "use_cache": true,
19 "bos_token_id": 151646,
20 "eos_token_id": 151643
21 }