config.json
803 B · 32 lines · json Raw
1 {
2 "_name_or_path": "/aot/checkpoints/15b/config.json",
3 "architectures": [
4 "Starcoder2ForCausalLM"
5 ],
6 "attention_dropout": 0.1,
7 "residual_dropout": 0.1,
8 "embedding_dropout": 0.1,
9 "bos_token_id": 0,
10 "eos_token_id": 0,
11 "hidden_act": "gelu_pytorch_tanh",
12 "hidden_size": 6144,
13 "initializer_range": 0.01275,
14 "intermediate_size": 24576,
15 "max_position_embeddings": 16384,
16 "mlp_type": "default",
17 "model_type": "starcoder2",
18 "norm_epsilon": 1e-05,
19 "norm_type": "layer_norm",
20 "num_attention_heads": 48,
21 "num_hidden_layers": 40,
22 "num_key_value_heads": 4,
23 "rope_theta": 100000,
24 "sliding_window": 4096,
25 "tie_word_embeddings": false,
26 "torch_dtype": "float32",
27 "transformers_version": "4.37.0.dev0",
28 "use_bias": true,
29 "use_cache": true,
30 "vocab_size": 49152
31 }
32