tokenizer_config.json
3.6 KB · 142 lines · json Raw
1 {
2 "audio_token": "<|audio|>",
3 "backend": "tokenizers",
4 "boa_token": "<|audio>",
5 "boi_token": "<|image>",
6 "bos_token": "<bos>",
7 "eoa_token": "<audio|>",
8 "eoc_token": "<channel|>",
9 "eoi_token": "<image|>",
10 "eos_token": "<turn|>",
11 "eot_token": "<turn|>",
12 "escape_token": "<|\"|>",
13 "etc_token": "<tool_call|>",
14 "etd_token": "<tool|>",
15 "etr_token": "<tool_response|>",
16 "extra_special_tokens": [
17 "<|video|>"
18 ],
19 "image_token": "<|image|>",
20 "is_local": true,
21 "mask_token": "<mask>",
22 "model_max_length": 1000000000000000019884624838656,
23 "model_specific_special_tokens": {
24 "audio_token": "<|audio|>",
25 "boa_token": "<|audio>",
26 "boi_token": "<|image>",
27 "eoa_token": "<audio|>",
28 "eoc_token": "<channel|>",
29 "eoi_token": "<image|>",
30 "eot_token": "<turn|>",
31 "escape_token": "<|\"|>",
32 "etc_token": "<tool_call|>",
33 "etd_token": "<tool|>",
34 "etr_token": "<tool_response|>",
35 "image_token": "<|image|>",
36 "soc_token": "<|channel>",
37 "sot_token": "<|turn>",
38 "stc_token": "<|tool_call>",
39 "std_token": "<|tool>",
40 "str_token": "<|tool_response>",
41 "think_token": "<|think|>"
42 },
43 "pad_token": "<pad>",
44 "padding_side": "left",
45 "processor_class": "Gemma4Processor",
46 "response_schema": {
47 "properties": {
48 "content": {
49 "type": "string"
50 },
51 "role": {
52 "const": "assistant"
53 },
54 "thinking": {
55 "type": "string"
56 },
57 "tool_calls": {
58 "items": {
59 "properties": {
60 "function": {
61 "properties": {
62 "arguments": {
63 "additionalProperties": {},
64 "type": "object",
65 "x-parser": "gemma4-tool-call"
66 },
67 "name": {
68 "type": "string"
69 }
70 },
71 "type": "object",
72 "x-regex": "call\\:(?P<name>\\w+)(?P<arguments>\\{.*\\})"
73 },
74 "type": {
75 "const": "function"
76 }
77 },
78 "type": "object"
79 },
80 "type": "array",
81 "x-regex-iterator": "<\\|tool_call>(.*?)<tool_call\\|>"
82 }
83 },
84 "type": "object",
85 "x-regex": "(\\<\\|channel\\>thought\\n(?P<thinking>.*?)\\<channel\\|\\>)?(?P<tool_calls>\\<\\|tool_call\\>.*\\<tool_call\\|\\>)?(?P<content>(?:(?!\\<turn\\|\\>)(?!\\<\\|tool_response\\>).)+)?(?:\\<turn\\|\\>|\\<\\|tool_response\\>)?"
86 },
87 "response_template": {
88 "defaults": {
89 "role": "assistant"
90 },
91 "fields": {
92 "content": {
93 "close": [
94 "<turn|>",
95 "<|tool_response>",
96 "<eos>"
97 ],
98 "content": "text"
99 },
100 "thinking": {
101 "close": "<channel|>",
102 "content": "text",
103 "open": "<|channel>thought\n"
104 },
105 "tool_calls": {
106 "close": "<tool_call|>",
107 "content": "json",
108 "content_args": {
109 "string_delims": [
110 [
111 "<|\"|>",
112 "<|\"|>"
113 ]
114 ],
115 "unquoted_keys": true
116 },
117 "open_pattern": "<\\|tool_call>call:(?P<name>\\w+)",
118 "repeats": true,
119 "transform": {
120 "function": {
121 "arguments": "{content}",
122 "name": "{name}"
123 },
124 "type": "function"
125 }
126 }
127 },
128 "start_anchor": [
129 "<|turn>model\n",
130 "<tool_response|>"
131 ]
132 },
133 "soc_token": "<|channel>",
134 "sot_token": "<|turn>",
135 "stc_token": "<|tool_call>",
136 "std_token": "<|tool>",
137 "str_token": "<|tool_response>",
138 "think_token": "<|think|>",
139 "tokenizer_class": "GemmaTokenizer",
140 "unk_token": "<unk>"
141 }
142