tokenizer_config.json
3.4 KB · 149 lines · json Raw
1 {
2 "add_prefix_space": false,
3 "audio_bos_token": "<|audio_start|>",
4 "audio_eos_token": "<|audio_end|>",
5 "audio_token": "<|audio_pad|>",
6 "backend": "tokenizers",
7 "bos_token": null,
8 "clean_up_tokenization_spaces": false,
9 "eos_token": "<|im_end|>",
10 "errors": "replace",
11 "extra_special_tokens": [
12 "<|image|>",
13 "<|video|>",
14 "<|sound|>",
15 "<|voice|>",
16 "<|listening|>",
17 "<|vision|>",
18 "<|mood_happy|>",
19 "<|mood_sad|>",
20 "<|mood_angry|>",
21 "<|mood_neutral|>",
22 "<fim_prefix>",
23 "<fim_middle>",
24 "<fim_suffix>",
25 "<|action_start|>",
26 "<|action_end|>",
27 "<|trajectory_start|>",
28 "<|trajectory_end|>",
29 "<|joint_start|>",
30 "<|joint_end|>",
31 "<|sensor_start|>",
32 "<|sensor_end|>",
33 "<|command_start|>",
34 "<|command_end|>",
35 "<|state_start|>",
36 "<|state_end|>",
37 "<|pose|>",
38 "<|velocity|>",
39 "<|force|>",
40 "<|torque|>",
41 "<|gripper|>",
42 "<|navigation|>",
43 "<|obstacle|>",
44 "<|task_start|>",
45 "<|task_end|>",
46 "<|plan_start|>",
47 "<|plan_end|>",
48 "<|behavior_start|>",
49 "<|behavior_end|>",
50 "<|skill_start|>",
51 "<|skill_end|>",
52 "<|motor|>",
53 "<|servo|>",
54 "<|imu|>",
55 "<|lidar|>",
56 "<|camera|>",
57 "<|depth|>",
58 "<|waypoint|>",
59 "<|path|>",
60 "<|collision|>",
61 "<|grasp|>",
62 "<|release|>",
63 "<|homing|>",
64 "<|emergency_stop|>",
65 "<|calibration|>",
66 "<|manipulation|>",
67 "<|locomotion|>",
68 "<|feedback|>",
69 "<|control_loop|>",
70 "<|language|>",
71 "<|tool_call_start|>",
72 "<|tool_call_end|>",
73 "<|tool_result_start|>",
74 "<|tool_result_end|>",
75 "__SCIENCE__",
76 "__CODING__",
77 "__STOCK_EXCHANGE__",
78 "__MEDICINE__",
79 "__GOVERNMENT__",
80 "__NEWS__",
81 "__GENERAL__",
82 "__MATERIAL_SCIENCE__",
83 "__ELECTRONICS__",
84 "__MICROELECTRONICS__",
85 "__ENGINEERING__",
86 "__ROBOTICS__",
87 "__ENERGY__",
88 "__AUTOMOTIVE__",
89 "__AVIATION__",
90 "__MATH__",
91 "__PYTHON__",
92 "__C__",
93 "__CPP__",
94 "__C_SHARP__",
95 "__JAVA__",
96 "__JAVASCRIPT__",
97 "__TYPESCRIPT__",
98 "__RUST__",
99 "__GO__",
100 "__RUBY__",
101 "__PHP__",
102 "__SWIFT__",
103 "__KOTLIN__",
104 "__BASH__",
105 "__SQL__",
106 "__ASSEMBLY__",
107 "__PHILOSOPHY__",
108 "__LITERATURE__",
109 "__SOCIOLOGY__",
110 "__PSYCHOLOGY__",
111 "__POLITICAL_SCIENCE__",
112 "__CULTURAL_STUDIES__",
113 "__ETHNOGRAPHY__",
114 "__HUMAN_RIGHTS__",
115 "__COMPLIANCE__",
116 "__MILITARY__",
117 "__BANKING__",
118 "__OIL_INDUSTRY__",
119 "__LIGHT_INDUSTRY__",
120 "__NATURE__",
121 "__OCEAN__",
122 "__SPORT__",
123 "__CULINARY__",
124 "__TRAVEL__",
125 "__HOBBY__"
126 ],
127 "image_token": "<|image_pad|>",
128 "is_local": false,
129 "local_files_only": false,
130 "model_max_length": 262144,
131 "model_specific_special_tokens": {
132 "audio_bos_token": "<|audio_start|>",
133 "audio_eos_token": "<|audio_end|>",
134 "audio_token": "<|audio_pad|>",
135 "image_token": "<|image_pad|>",
136 "video_token": "<|video_pad|>",
137 "vision_bos_token": "<|vision_start|>",
138 "vision_eos_token": "<|vision_end|>"
139 },
140 "pad_token": "<|endoftext|>",
141 "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
142 "split_special_tokens": false,
143 "tokenizer_class": "Qwen2Tokenizer",
144 "unk_token": null,
145 "video_token": "<|video_pad|>",
146 "vision_bos_token": "<|vision_start|>",
147 "vision_eos_token": "<|vision_end|>"
148 }
149