config.json
1.3 KB · 49 lines · json Raw
1 {
2 "architectures": [
3 "ModernBertModel"
4 ],
5 "attention_bias": false,
6 "attention_dropout": 0.0,
7 "attention_probs_dropout_prob": 0.1,
8 "bos_token_id": 50281,
9 "classifier_activation": "silu",
10 "classifier_bias": false,
11 "classifier_dropout": 0.0,
12 "classifier_pooling": "mean",
13 "cls_token_id": 50281,
14 "decoder_bias": true,
15 "deterministic_flash_attn": false,
16 "embedding_dropout": 0.0,
17 "eos_token_id": 50282,
18 "global_attn_every_n_layers": 3,
19 "global_rope_theta": 80000,
20 "gradient_checkpointing": false,
21 "hidden_act": "gelu",
22 "hidden_activation": "gelu",
23 "hidden_dropout_prob": 0.1,
24 "hidden_size": 384,
25 "initializer_cutoff_factor": 2.0,
26 "initializer_range": 0.02,
27 "intermediate_size": 1536,
28 "layer_norm_eps": 1e-05,
29 "local_attention": 128,
30 "local_rope_theta": 10000.0,
31 "max_position_embeddings": 8192,
32 "mlp_bias": false,
33 "mlp_dropout": 0.0,
34 "model_type": "modernbert",
35 "norm_bias": false,
36 "norm_eps": 1e-05,
37 "num_attention_heads": 12,
38 "num_hidden_layers": 12,
39 "pad_token_id": 50283,
40 "position_embedding_type": "relative_key_query",
41 "repad_logits_with_grad": false,
42 "sep_token_id": 50282,
43 "sparse_pred_ignore_index": -100,
44 "sparse_prediction": false,
45 "torch_dtype": "bfloat16",
46 "transformers_version": "4.51.3",
47 "vocab_size": 50368
48 }
49