multilingual/encoder/config.json
1.9 KB · 80 lines · json Raw
1 {
2 "architectures": [
3 "ModernBertForMaskedLM"
4 ],
5 "attention_bias": false,
6 "attention_dropout": 0.0,
7 "bos_token_id": 2,
8 "classifier_activation": "gelu",
9 "classifier_bias": false,
10 "classifier_dropout": 0.0,
11 "classifier_pooling": "mean",
12 "cls_token_id": 1,
13 "decoder_bias": true,
14 "deterministic_flash_attn": false,
15 "dtype": "float32",
16 "embedding_dropout": 0.0,
17 "eos_token_id": 1,
18 "global_attn_every_n_layers": 3,
19 "gradient_checkpointing": false,
20 "hidden_activation": "gelu",
21 "hidden_size": 768,
22 "initializer_cutoff_factor": 2.0,
23 "initializer_range": 0.02,
24 "intermediate_size": 1152,
25 "layer_norm_eps": 1e-05,
26 "layer_types": [
27 "full_attention",
28 "sliding_attention",
29 "sliding_attention",
30 "full_attention",
31 "sliding_attention",
32 "sliding_attention",
33 "full_attention",
34 "sliding_attention",
35 "sliding_attention",
36 "full_attention",
37 "sliding_attention",
38 "sliding_attention",
39 "full_attention",
40 "sliding_attention",
41 "sliding_attention",
42 "full_attention",
43 "sliding_attention",
44 "sliding_attention",
45 "full_attention",
46 "sliding_attention",
47 "sliding_attention",
48 "full_attention"
49 ],
50 "local_attention": 128,
51 "mask_token_id": 4,
52 "max_position_embeddings": 8192,
53 "mlp_bias": false,
54 "mlp_dropout": 0.0,
55 "model_type": "modernbert",
56 "norm_bias": false,
57 "norm_eps": 1e-05,
58 "num_attention_heads": 12,
59 "num_hidden_layers": 22,
60 "pad_token_id": 0,
61 "position_embedding_type": "sans_pos",
62 "repad_logits_with_grad": false,
63 "rope_parameters": {
64 "full_attention": {
65 "rope_theta": 160000,
66 "rope_type": "default"
67 },
68 "sliding_attention": {
69 "rope_theta": 160000,
70 "rope_type": "default"
71 }
72 },
73 "sep_token_id": 1,
74 "sparse_pred_ignore_index": -100,
75 "sparse_prediction": false,
76 "tie_word_embeddings": true,
77 "transformers_version": "5.0.0",
78 "vocab_size": 256000
79 }
80