special_tokens_map.json
962 B · 52 lines · json Raw
1 {
2 "bos_token": {
3 "content": "<s>",
4 "lstrip": false,
5 "normalized": false,
6 "rstrip": false,
7 "single_word": false
8 },
9 "cls_token": {
10 "content": "<s>",
11 "lstrip": false,
12 "normalized": true,
13 "rstrip": false,
14 "single_word": false
15 },
16 "eos_token": {
17 "content": "</s>",
18 "lstrip": false,
19 "normalized": false,
20 "rstrip": false,
21 "single_word": false
22 },
23 "mask_token": {
24 "content": "<mask>",
25 "lstrip": true,
26 "normalized": false,
27 "rstrip": false,
28 "single_word": false
29 },
30 "pad_token": {
31 "content": "<pad>",
32 "lstrip": false,
33 "normalized": false,
34 "rstrip": false,
35 "single_word": false
36 },
37 "sep_token": {
38 "content": "</s>",
39 "lstrip": false,
40 "normalized": true,
41 "rstrip": false,
42 "single_word": false
43 },
44 "unk_token": {
45 "content": "[UNK]",
46 "lstrip": false,
47 "normalized": false,
48 "rstrip": false,
49 "single_word": false
50 }
51 }
52