config.json
1.3 KB · 62 lines · json Raw
1 {
2 "architectures": [
3 "DSparkDraftModel"
4 ],
5 "attention_bias": false,
6 "attention_dropout": 0.0,
7 "auto_map": {
8 "AutoModel": "dspark.DSparkDraftModel"
9 },
10 "block_size": 7,
11 "bos_token_id": 163584,
12 "confidence_head_with_markov": true,
13 "dflash_config": {
14 "mask_token_id": 163824,
15 "target_layer_ids": [
16 7,
17 23,
18 51,
19 67,
20 83
21 ]
22 },
23 "dtype": "bfloat16",
24 "enable_confidence_head": true,
25 "eos_token_id": 163586,
26 "head_dim": 64,
27 "hidden_act": "silu",
28 "hidden_size": 7168,
29 "initializer_range": 0.02,
30 "intermediate_size": 14336,
31 "layer_types": [
32 "full_attention",
33 "full_attention",
34 "full_attention",
35 "full_attention",
36 "full_attention"
37 ],
38 "markov_head_type": "vanilla",
39 "markov_rank": 256,
40 "max_position_embeddings": 1048576,
41 "max_window_layers": 5,
42 "model_type": "qwen3",
43 "num_attention_heads": 64,
44 "num_hidden_layers": 5,
45 "num_key_value_heads": 16,
46 "num_target_layers": 93,
47 "pad_token_id": 163839,
48 "rms_norm_eps": 1e-05,
49 "rope_parameters": {
50 "factor": 16.0,
51 "original_max_position_embeddings": 65536,
52 "rope_theta": 10000.0,
53 "rope_type": "yarn"
54 },
55 "sliding_window": null,
56 "tie_word_embeddings": false,
57 "transformers_version": "5.12.1",
58 "use_cache": true,
59 "use_sliding_window": false,
60 "vocab_size": 163840
61 }
62