config.json
2.7 KB · 113 lines · json Raw
1 {
2 "_hidden_size": 2560,
3 "_vocab_size": 151936,
4 "architectures": [
5 "HiggsMultimodalQwen3ForConditionalGeneration"
6 ],
7 "audio_encoder_config": {
8 "_name_or_path": "",
9 "architectures": null,
10 "chunk_size_feed_forward": 0,
11 "dtype": null,
12 "encoder_type": "discrete",
13 "id2label": {
14 "0": "LABEL_0",
15 "1": "LABEL_1"
16 },
17 "is_encoder_decoder": false,
18 "label2id": {
19 "LABEL_0": 0,
20 "LABEL_1": 1
21 },
22 "max_chunk_size": 50,
23 "mel_per_sample": 8,
24 "model_type": "higgs_audio_encoder",
25 "num_codebooks": 8,
26 "out_dim": 2560,
27 "output_attentions": false,
28 "output_hidden_states": false,
29 "problem_type": null,
30 "qwen3_aut_config": null,
31 "return_dict": true,
32 "tie_word_embeddings": true,
33 "use_delay_pattern": true,
34 "vocab_size": 1026,
35 "whisper_config": null
36 },
37 "audio_token_id": -100,
38 "ignore_index": -100,
39 "model_type": "higgs_multimodal_qwen3",
40 "text_config": {
41 "_name_or_path": "/ceph/models/Qwen3-4B-Base",
42 "architectures": [
43 "Qwen3ForCausalLM"
44 ],
45 "attention_bias": false,
46 "attention_dropout": 0.0,
47 "bos_token_id": 151643,
48 "dtype": "bfloat16",
49 "eos_token_id": 151643,
50 "head_dim": 128,
51 "hidden_act": "silu",
52 "hidden_size": 2560,
53 "initializer_range": 0.02,
54 "intermediate_size": 9728,
55 "layer_types": [
56 "full_attention",
57 "full_attention",
58 "full_attention",
59 "full_attention",
60 "full_attention",
61 "full_attention",
62 "full_attention",
63 "full_attention",
64 "full_attention",
65 "full_attention",
66 "full_attention",
67 "full_attention",
68 "full_attention",
69 "full_attention",
70 "full_attention",
71 "full_attention",
72 "full_attention",
73 "full_attention",
74 "full_attention",
75 "full_attention",
76 "full_attention",
77 "full_attention",
78 "full_attention",
79 "full_attention",
80 "full_attention",
81 "full_attention",
82 "full_attention",
83 "full_attention",
84 "full_attention",
85 "full_attention",
86 "full_attention",
87 "full_attention",
88 "full_attention",
89 "full_attention",
90 "full_attention",
91 "full_attention"
92 ],
93 "max_position_embeddings": 32768,
94 "max_window_layers": 36,
95 "model_type": "qwen3",
96 "num_attention_heads": 32,
97 "num_hidden_layers": 36,
98 "num_key_value_heads": 8,
99 "pad_token_id": null,
100 "rms_norm_eps": 1e-06,
101 "rope_parameters": {
102 "rope_theta": 1000000,
103 "rope_type": "default"
104 },
105 "sliding_window": null,
106 "tie_word_embeddings": true,
107 "use_cache": true,
108 "use_sliding_window": false,
109 "vocab_size": 151936
110 },
111 "transformers_version": "5.5.0"
112 }
113