config.json
1.8 KB · 72 lines · json Raw
1 {
2 "architectures": [
3 "DeepseekV4ForCausalLM"
4 ],
5 "attention_bias": false,
6 "attention_dropout": 0.0,
7 "bos_token_id": 0,
8 "eos_token_id": 1,
9 "expert_dtype": "fp4",
10 "hc_eps": 1e-06,
11 "hc_mult": 4,
12 "hc_sinkhorn_iters": 20,
13 "head_dim": 512,
14 "hidden_act": "silu",
15 "hidden_size": 4096,
16 "index_head_dim": 128,
17 "index_n_heads": 64,
18 "index_topk": 512,
19 "initializer_range": 0.02,
20 "max_position_embeddings": 1048576,
21 "model_type": "deepseek_v4",
22 "moe_intermediate_size": 2048,
23 "n_routed_experts": 256,
24 "n_shared_experts": 1,
25 "norm_topk_prob": true,
26 "num_attention_heads": 64,
27 "num_experts_per_tok": 6,
28 "num_hidden_layers": 43,
29 "num_hash_layers": 3,
30 "num_key_value_heads": 1,
31 "num_nextn_predict_layers": 1,
32 "o_groups": 8,
33 "o_lora_rank": 1024,
34 "q_lora_rank": 1024,
35 "qk_rope_head_dim": 64,
36 "quantization_config": {
37 "activation_scheme": "dynamic",
38 "fmt": "e4m3",
39 "quant_method": "fp8",
40 "scale_fmt": "ue8m0",
41 "weight_block_size": [
42 128,
43 128
44 ]
45 },
46 "rms_norm_eps": 1e-06,
47 "rope_scaling": {
48 "beta_fast": 32,
49 "beta_slow": 1,
50 "factor": 16,
51 "original_max_position_embeddings": 65536,
52 "type": "yarn"
53 },
54 "rope_theta": 10000,
55 "routed_scaling_factor": 1.5,
56 "scoring_func": "sqrtsoftplus",
57 "sliding_window": 128,
58 "swiglu_limit": 10.0,
59 "tie_word_embeddings": false,
60 "topk_method": "noaux_tc",
61 "torch_dtype": "bfloat16",
62 "transformers_version": "4.57.1",
63 "use_cache": true,
64 "vocab_size": 129280,
65 "compress_rope_theta": 160000,
66 "compress_ratios": [0, 0, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 0, 0, 0],
67 "dspark_block_size": 5,
68 "dspark_noise_token_id": 128799,
69 "dspark_target_layer_ids": [40, 41, 42],
70 "dspark_markov_rank": 256
71 }
72