tokenizer_config.json
| 1 | { |
| 2 | "add_bos_token": true, |
| 3 | "add_eos_token": false, |
| 4 | "bos_token": { |
| 5 | "__type": "AddedToken", |
| 6 | "content": "<｜begin▁of▁sentence｜>", |
| 7 | "lstrip": false, |
| 8 | "normalized": true, |
| 9 | "rstrip": false, |
| 10 | "single_word": false |
| 11 | }, |
| 12 | "clean_up_tokenization_spaces": false, |
| 13 | "eos_token": { |
| 14 | "__type": "AddedToken", |
| 15 | "content": "<｜end▁of▁sentence｜>", |
| 16 | "lstrip": false, |
| 17 | "normalized": true, |
| 18 | "rstrip": false, |
| 19 | "single_word": false |
| 20 | }, |
| 21 | "legacy": true, |
| 22 | "model_max_length": 16384, |
| 23 | "pad_token": { |
| 24 | "__type": "AddedToken", |
| 25 | "content": "<｜end▁of▁sentence｜>", |
| 26 | "lstrip": false, |
| 27 | "normalized": true, |
| 28 | "rstrip": false, |
| 29 | "single_word": false |
| 30 | }, |
| 31 | "sp_model_kwargs": {}, |
| 32 | "unk_token": null, |
| 33 | "tokenizer_class": "LlamaTokenizerFast", |
| 34 | "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜><think>\\n'}}{% endif %}" |
| 35 | } |