special_tokens_map.json
799 B · 41 lines · json Raw
1 {
2 "additional_special_tokens": [
3 "<|endoftext|>",
4 "[MASK]",
5 "[gMASK]",
6 "[sMASK]",
7 "<sop>",
8 "<eop>",
9 "<|system|>",
10 "<|user|>",
11 "<|assistant|>",
12 "<|observation|>",
13 "<|begin_of_image|>",
14 "<|end_of_image|>",
15 "<|begin_of_video|>",
16 "<|end_of_video|>",
17 "<|begin_of_audio|>",
18 "<|end_of_audio|>",
19 "<|begin_of_transcription|>",
20 "<|end_of_transcription|>",
21 "<|code_prefix|>",
22 "<|code_middle|>",
23 "<|code_suffix|>",
24 "/nothink"
25 ],
26 "eos_token": {
27 "content": "<|im_end|>",
28 "lstrip": false,
29 "normalized": false,
30 "rstrip": false,
31 "single_word": false
32 },
33 "pad_token": {
34 "content": "<|endoftext|>",
35 "lstrip": false,
36 "normalized": false,
37 "rstrip": false,
38 "single_word": false
39 }
40 }
41