jon-tow commited on
Commit
542dce2
·
verified ·
1 Parent(s): c89d7d1

tmpfix(tokenizer_config): force `GPT2TokenizerFast`

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -39,7 +39,7 @@
39
  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
40
  "clean_up_tokenization_spaces": true,
41
  "eos_token": "<|endoftext|>",
42
- "tokenizer_class": "GPT2Tokenizer",
43
  "model_max_length": 2048,
44
  "pad_token": "<|endoftext|>",
45
  "unk_token": "<|endoftext|>"
 
39
  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
40
  "clean_up_tokenization_spaces": true,
41
  "eos_token": "<|endoftext|>",
42
+ "tokenizer_class": "GPT2TokenizerFast",
43
  "model_max_length": 2048,
44
  "pad_token": "<|endoftext|>",
45
  "unk_token": "<|endoftext|>"