config.json
1.4 KB · 55 lines · json Raw
1 {
2 "architectures": [
3 "NemotronHForCausalLM"
4 ],
5 "attention_bias": false,
6 "attention_dropout": 0.0,
7 "head_dim": 128,
8 "auto_map": {
9 "AutoConfig": "configuration_nemotron_h.NemotronHConfig",
10 "AutoModelForCausalLM": "modeling_nemotron_h.NemotronHForCausalLM"
11 },
12 "bos_token_id": 1,
13 "chunk_size": 256,
14 "conv_kernel": 4,
15 "eos_token_id": 2,
16 "expand": 2,
17 "hidden_dropout": 0.0,
18 "hidden_size": 3136,
19 "hybrid_override_pattern": "M-M-M-MM-M-M*-M-M*-M-M-M*-M-M-MM*-MMM-M-M-",
20 "initializer_range": 0.02,
21 "intermediate_size": 12544,
22 "layer_norm_epsilon": 1e-05,
23 "mamba_head_dim": 80,
24 "mamba_hidden_act": "silu",
25 "mamba_num_heads": 96,
26 "mamba_proj_bias": false,
27 "max_position_embeddings": 262144,
28 "mlp_bias": false,
29 "mlp_hidden_act": "relu2",
30 "model_type": "nemotron_h",
31 "n_groups": 8,
32 "num_attention_heads": 40,
33 "num_hidden_layers": 42,
34 "num_key_value_heads": 8,
35 "num_logits_to_keep": 1,
36 "pad_token_id": 0,
37 "rescale_prenorm_residual": true,
38 "residual_in_fp32": false,
39 "rms_norm_eps": 1e-05,
40 "sliding_window": null,
41 "ssm_state_size": 128,
42 "tie_word_embeddings": false,
43 "time_step_floor": 0.0001,
44 "time_step_max": 0.1,
45 "time_step_min": 0.001,
46 "time_step_rank": 256,
47 "torch_dtype": "bfloat16",
48 "transformers_version": "4.53.0",
49 "use_bias": false,
50 "use_cache": true,
51 "use_conv_bias": true,
52 "use_mamba_kernels": true,
53 "vocab_size": 131072
54 }
55