config.json
865 B · 37 lines · json Raw
1 {
2 "architectures": [
3 "LlamaForCausalLM"
4 ],
5 "attention_bias": false,
6 "attention_dropout": 0.0,
7 "bos_token_id": 128000,
8 "eos_token_id": 128009,
9 "head_dim": 128,
10 "hidden_act": "silu",
11 "hidden_size": 3072,
12 "initializer_range": 0.02,
13 "intermediate_size": 8192,
14 "max_position_embeddings": 131072,
15 "mlp_bias": false,
16 "model_type": "llama",
17 "num_attention_heads": 24,
18 "num_hidden_layers": 28,
19 "num_key_value_heads": 8,
20 "pad_token_id": 128256,
21 "pretraining_tp": 1,
22 "rms_norm_eps": 1e-05,
23 "rope_scaling": {
24 "factor": 32.0,
25 "high_freq_factor": 4.0,
26 "low_freq_factor": 1.0,
27 "original_max_position_embeddings": 8192,
28 "rope_type": "llama3"
29 },
30 "rope_theta": 500000.0,
31 "tie_word_embeddings": true,
32 "torch_dtype": "bfloat16",
33 "transformers_version": "4.52.1",
34 "use_cache": true,
35 "vocab_size": 128257
36 }
37