special_tokens_map.json
1.6 KB · 82 lines · json Raw
1 {
2 "additional_special_tokens": [
3 {
4 "content": "<|im_end|>",
5 "lstrip": false,
6 "normalized": false,
7 "rstrip": false,
8 "single_word": false
9 },
10 {
11 "content": "<|im_start|>",
12 "lstrip": false,
13 "normalized": false,
14 "rstrip": false,
15 "single_word": false
16 },
17 {
18 "content": "<|tool_call|>",
19 "lstrip": false,
20 "normalized": false,
21 "rstrip": false,
22 "single_word": false
23 },
24 {
25 "content": "<|execute_start|>",
26 "lstrip": false,
27 "normalized": false,
28 "rstrip": false,
29 "single_word": false
30 },
31 {
32 "content": "<|execute_end|>",
33 "lstrip": false,
34 "normalized": false,
35 "rstrip": false,
36 "single_word": false
37 },
38 {
39 "content": "<|fim_prefix|>",
40 "lstrip": false,
41 "normalized": false,
42 "rstrip": false,
43 "single_word": false
44 },
45 {
46 "content": "<|fim_middle|>",
47 "lstrip": false,
48 "normalized": false,
49 "rstrip": false,
50 "single_word": false
51 },
52 {
53 "content": "<|fim_suffix|>",
54 "lstrip": false,
55 "normalized": false,
56 "rstrip": false,
57 "single_word": false
58 }
59 ],
60 "bos_token": {
61 "content": "<s>",
62 "lstrip": false,
63 "normalized": false,
64 "rstrip": false,
65 "single_word": false
66 },
67 "eos_token": {
68 "content": "</s>",
69 "lstrip": false,
70 "normalized": false,
71 "rstrip": false,
72 "single_word": false
73 },
74 "unk_token": {
75 "content": "<unk>",
76 "lstrip": false,
77 "normalized": false,
78 "rstrip": false,
79 "single_word": false
80 }
81 }
82