config.json
705 B · 30 lines · json Raw
1 {
2 "_name_or_path": "01-ai/Yi-1.5-34B",
3 "architectures": [
4 "LlamaForCausalLM"
5 ],
6 "attention_bias": false,
7 "attention_dropout": 0.0,
8 "bos_token_id": 1,
9 "eos_token_id": 7,
10 "hidden_act": "silu",
11 "hidden_size": 7168,
12 "initializer_range": 0.02,
13 "intermediate_size": 20480,
14 "max_position_embeddings": 8192,
15 "model_type": "llama",
16 "num_attention_heads": 56,
17 "num_hidden_layers": 60,
18 "num_key_value_heads": 8,
19 "pad_token_id": 0,
20 "pretraining_tp": 1,
21 "rms_norm_eps": 1e-06,
22 "rope_scaling": null,
23 "rope_theta": 5000000.0,
24 "tie_word_embeddings": false,
25 "torch_dtype": "bfloat16",
26 "transformers_version": "4.40.0.dev0",
27 "use_cache": false,
28 "vocab_size": 64000
29 }
30