special_tokens_map.json
670 B · 37 lines · json Raw
1 {
2 "additional_special_tokens": [
3 "<|system|>",
4 "<|end|>",
5 "<|user|>",
6 "<|end|>"
7 ],
8 "bos_token": {
9 "content": "<s>",
10 "lstrip": false,
11 "normalized": false,
12 "rstrip": false,
13 "single_word": false
14 },
15 "eos_token": {
16 "content": "<|endoftext|>",
17 "lstrip": false,
18 "normalized": false,
19 "rstrip": false,
20 "single_word": false
21 },
22 "pad_token": {
23 "content": "<|endoftext|>",
24 "lstrip": false,
25 "normalized": false,
26 "rstrip": false,
27 "single_word": false
28 },
29 "unk_token": {
30 "content": "<unk>",
31 "lstrip": false,
32 "normalized": false,
33 "rstrip": false,
34 "single_word": false
35 }
36 }
37