tokenizer_config.json
| 1 | { |
| 2 | "add_prefix_space": false, |
| 3 | "bos_token": "<|endoftext|>", |
| 4 | "eos_token": "<|endoftext|>", |
| 5 | "name_or_path": "EleutherAI/gpt-neox-20b", |
| 6 | "special_tokens_map_file": "/admin/home-hailey/.cache/huggingface/hub/models--EleutherAI--gpt-neox-20b/snapshots/4e49eadb5d14bd22f314ec3f45b69a87b88c7691/special_tokens_map.json", |
| 7 | "tokenizer_class": "GPTNeoXTokenizer", |
| 8 | "unk_token": "<|endoftext|>" |
| 9 | } |
| 10 | |