special_tokens_map.json
636 B · 35 lines · json Raw
1 {
2 "additional_special_tokens": [
3 "<start_of_turn>",
4 "<end_of_turn>"
5 ],
6 "bos_token": {
7 "content": "<bos>",
8 "lstrip": false,
9 "normalized": false,
10 "rstrip": false,
11 "single_word": false
12 },
13 "eos_token": {
14 "content": "<eos>",
15 "lstrip": false,
16 "normalized": false,
17 "rstrip": false,
18 "single_word": false
19 },
20 "pad_token": {
21 "content": "<pad>",
22 "lstrip": false,
23 "normalized": false,
24 "rstrip": false,
25 "single_word": false
26 },
27 "unk_token": {
28 "content": "<unk>",
29 "lstrip": false,
30 "normalized": false,
31 "rstrip": false,
32 "single_word": false
33 }
34 }
35