multilingual/tokenizer/tokenizer_config.json
| 1 | { |
| 2 | "bos_token": "<bos>", |
| 3 | "clean_up_tokenization_spaces": false, |
| 4 | "cls_token": "<bos>", |
| 5 | "eos_token": "<eos>", |
| 6 | "extra_special_tokens": { |
| 7 | "extra_0": "<start_of_turn>", |
| 8 | "extra_1": "<end_of_turn>" |
| 9 | }, |
| 10 | "mask_token": "<mask>", |
| 11 | "model_input_names": [ |
| 12 | "input_ids", |
| 13 | "attention_mask" |
| 14 | ], |
| 15 | "model_max_length": 8192, |
| 16 | "pad_token": "<pad>", |
| 17 | "padding_side": "right", |
| 18 | "sep_token": "<eos>", |
| 19 | "spaces_between_special_tokens": false, |
| 20 | "tokenizer_class": "PreTrainedTokenizerFast", |
| 21 | "unk_token": "<unk>" |
| 22 | } |