From e5c98976c22b155954a525a8b04e7c61e6ba8cf0 Mon Sep 17 00:00:00 2001
From: ai-modelscope <ai@modelscope.cn>
Date: Wed, 18 Sep 2024 11:18:28 +0800
Subject: [PATCH] update tokenizer_config

---
 .gitattributes        | 13 +------------
 tokenizer_config.json |  4 ++--
 2 files changed, 3 insertions(+), 14 deletions(-)

diff --git a/.gitattributes b/.gitattributes
index 8984f08..39e7ae7 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -32,15 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.xz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
-model-00001-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
-model-00002-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
-model-00003-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
-model-00004-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
-model-00005-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
-model-00006-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
-model-00007-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
-model-00008-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
-model-00009-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
-model-00010-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
-model-00011-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
\ No newline at end of file
diff --git a/tokenizer_config.json b/tokenizer_config.json
index ea25edd..1cad147 100644
--- a/tokenizer_config.json
+++ b/tokenizer_config.json
@@ -195,7 +195,7 @@
     "<|video_pad|>"
   ],
   "bos_token": null,
-  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
   "errors": "replace",
@@ -204,4 +204,4 @@
   "split_special_tokens": false,
   "tokenizer_class": "Qwen2Tokenizer",
   "unk_token": null
-}
+}
\ No newline at end of file