tokenizer_config.json
2.7 KB · 142 lines · json Raw
1 {
2 "add_prefix_space": false,
3 "audio_bos_token": "<|audio_start|>",
4 "audio_eos_token": "<|audio_end|>",
5 "audio_token": "<|audio_pad|>",
6 "backend": "tokenizers",
7 "bos_token": null,
8 "clean_up_tokenization_spaces": false,
9 "eos_token": "<|im_end|>",
10 "errors": "replace",
11 "extra_special_tokens": [
12 "<obj0>",
13 "<obj1>",
14 "<obj2>",
15 "<obj3>",
16 "<obj4>",
17 "<obj5>",
18 "<obj6>",
19 "<obj7>",
20 "<obj8>",
21 "<obj9>",
22 "<obj10>",
23 "<obj11>",
24 "<obj12>",
25 "<obj13>",
26 "<obj14>",
27 "<obj15>",
28 "<obj16>",
29 "<obj17>",
30 "<obj18>",
31 "<obj19>",
32 "<obj20>",
33 "<obj21>",
34 "<obj22>",
35 "<obj23>",
36 "<obj24>",
37 "<obj25>",
38 "<obj26>",
39 "<obj27>",
40 "<obj28>",
41 "<obj29>",
42 "<obj30>",
43 "<obj31>",
44 "<obj32>",
45 "<obj33>",
46 "<obj34>",
47 "<obj35>",
48 "<obj36>",
49 "<obj37>",
50 "<obj38>",
51 "<obj39>",
52 "<obj40>",
53 "<obj41>",
54 "<obj42>",
55 "<obj43>",
56 "<obj44>",
57 "<obj45>",
58 "<obj46>",
59 "<obj47>",
60 "<obj48>",
61 "<obj49>",
62 "<obj50>",
63 "<obj51>",
64 "<obj52>",
65 "<obj53>",
66 "<obj54>",
67 "<obj55>",
68 "<obj56>",
69 "<obj57>",
70 "<obj58>",
71 "<obj59>",
72 "<obj60>",
73 "<obj61>",
74 "<obj62>",
75 "<obj63>",
76 "<obj64>",
77 "<obj65>",
78 "<obj66>",
79 "<obj67>",
80 "<obj68>",
81 "<obj69>",
82 "<obj70>",
83 "<obj71>",
84 "<obj72>",
85 "<obj73>",
86 "<obj74>",
87 "<obj75>",
88 "<obj76>",
89 "<obj77>",
90 "<obj78>",
91 "<obj79>",
92 "<obj80>",
93 "<obj81>",
94 "<obj82>",
95 "<obj83>",
96 "<obj84>",
97 "<obj85>",
98 "<obj86>",
99 "<obj87>",
100 "<obj88>",
101 "<obj89>",
102 "<obj90>",
103 "<obj91>",
104 "<obj92>",
105 "<obj93>",
106 "<obj94>",
107 "<obj95>",
108 "<obj96>",
109 "<obj97>",
110 "<obj98>",
111 "<obj99>",
112 "<ground>",
113 "</ground>",
114 "<objects>",
115 "</objects>",
116 "<objfeat>",
117 "<think>",
118 "</think>"
119 ],
120 "image_token": "<|image_pad|>",
121 "is_local": true,
122 "model_max_length": 8192,
123 "model_specific_special_tokens": {
124 "audio_bos_token": "<|audio_start|>",
125 "audio_eos_token": "<|audio_end|>",
126 "audio_token": "<|audio_pad|>",
127 "image_token": "<|image_pad|>",
128 "video_token": "<|video_pad|>",
129 "vision_bos_token": "<|vision_start|>",
130 "vision_eos_token": "<|vision_end|>"
131 },
132 "pad_token": "<|endoftext|>",
133 "padding_side": "right",
134 "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
135 "split_special_tokens": false,
136 "tokenizer_class": "TokenizersBackend",
137 "unk_token": null,
138 "video_token": "<|video_pad|>",
139 "vision_bos_token": "<|vision_start|>",
140 "vision_eos_token": "<|vision_end|>"
141 }
142