public/model-minimax-m2
62
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
ad7ace91bd
model-minimax-m2/tokenizer_config.json
kelseye 2caf9c4697 Upload folder using ModelScope SDK
2025-10-26 08:46:06 +00:00

496 lines
11 KiB
JSON
Raw Blame History

{
"added_tokens_decoder": {
"200000": {
"content": "]!p~[",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200001": {
"content": "<fim_prefix>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200002": {
"content": "<fim_middle>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200003": {
"content": "<fim_suffix>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200004": {
"content": "<fim_pad>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200005": {
"content": "<reponame>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200006": {
"content": "<filename>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200007": {
"content": "<gh_stars>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200008": {
"content": "<issue_start>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200009": {
"content": "<issue_comment>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200010": {
"content": "<issue_closed>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200011": {
"content": "<jupyter_start>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200012": {
"content": "<jupyter_text>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200013": {
"content": "<jupyter_code>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200014": {
"content": "<jupyter_output>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200015": {
"content": "<empty_output>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200016": {
"content": "<commit_before>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200017": {
"content": "<commit_msg>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200018": {
"content": "<commit_after>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200019": {
"content": "]~b]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200020": {
"content": "[e~[",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200021": {
"content": "]!d~[",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200022": {
"content": "<function_call>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200023": {
"content": "<code_interpreter>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200024": {
"content": "]<]speech[>[",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200025": {
"content": "]<]image[>[",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200026": {
"content": "]<]video[>[",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200027": {
"content": "]<]start of speech[>[",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200028": {
"content": "]<]end of speech[>[",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200029": {
"content": "]<]start of image[>[",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200030": {
"content": "]<]end of image[>[",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200031": {
"content": "]<]start of video[>[",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200032": {
"content": "]<]end of video[>[",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200033": {
"content": "]<]vision pad[>[",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200034": {
"content": "]~!b[",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200035": {
"content": "<jupyter_error>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200036": {
"content": "<add_file>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
"200037": {
"content": "<delete_file>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200038": {
"content": "<rename_file>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200039": {
"content": "<edit_file>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200040": {
"content": "<commit_message>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200041": {
"content": "<empty_source_file>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200042": {
"content": "<repo_struct>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"200043": {
"content": "<code_context>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
"200044": {
"content": "<file_content>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
"200045": {
"content": "<source_files>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
"200046": {
"content": "<pr_start>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
"200047": {
"content": "<review_comment>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
"200048": {
"content": "<filepath>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
"200049": {
"content": "<file_sep>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
"200050": {
"content": "<think>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": false
},
"200051": {
"content": "</think>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": false
},
"200052": {
"content": "<minimax:tool_call>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": false
},
"200053": {
"content": "</minimax:tool_call>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": false
}
},
"additional_special_tokens": [
"<code_interpreter>",
"<commit_after>",
"<commit_before>",
"<commit_msg>",
"<empty_output>",
"<filename>",
"<fim_middle>",
"<fim_pad>",
"<fim_prefix>",
"<fim_suffix>",
"<function_call>",
"<gh_stars>",
"]<]speech[>[",
"]<]image[>[",
"]<]video[>[",
"]<]start of speech[>[",
"]<]end of speech[>[",
"]<]start of image[>[",
"]<]end of image[>[",
"]<]start of video[>[",
"]<]end of video[>[",
"]<]vision pad[>[",
"]~!b[",
"<issue_closed>",
"<issue_comment>",
"<issue_start>",
"<jupyter_code>",
"<jupyter_output>",
"<jupyter_start>",
"<jupyter_text>",
"<reponame>",
"[e~[",
"]!d~[",
"]!p~[",
"]~b]",
"<jupyter_error>",
"<add_file>",
"<delete_file>",
"<rename_file>",
"<edit_file>",
"<commit_message>",
"<empty_source_file>",
"<repo_struct>",
"<code_context>",
"<file_content>",
"<source_files>",
"<pr_start>",
"<review_comment>",
"<filepath>",
"<file_sep>"
],
"add_prefix_space": false,
"bos_token": "]~!b[",
"clean_up_tokenization_spaces": false,
"eos_token": "[e~[",
"model_max_length": 40960000,
"tokenizer_class": "GPT2Tokenizer",
"unk_token": "]!d~["
}
Reference in New Issue View Git Blame Copy Permalink