public/model-chatterbox-multilingual-onnx
62
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Upload tokenizer.json

Browse Source
This commit is contained in:
Cherrytest 2025-09-30 03:55:57 +00:00
parent 985eb49a65
commit 97323c8c35
1 changed files with 280 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

331
tokenizer.json
View File

@ -21,15 +21,6 @@
"rstrip": false, "rstrip": false,
"normalized": false "normalized": false
}, },
{
"id": 2,
"special": true,
"content": "[SPACE]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true
},
{ {
"id": 255, "id": 255,
"special": true, "special": true,
@ -1064,69 +1055,199 @@
"rstrip": false, "rstrip": false,
"normalized": false, "normalized": false,
"special": true "special": true
},
{
"id": 6561,
"content": "[START_SPEECH]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 6562,
"content": "[STOP_SPEECH]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 6563,
"content": "[EXAGGERATION]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
} }
], ],
"normalizer": { "normalizer": {
"type": "Sequence",
"normalizers": [
{
"type": "Replace", "type": "Replace",
"pattern": { "String": " " }, "pattern": {
"content": "[SPACE]" "Regex": "\\s+"
}
]
}, },
"pre_tokenizer": { "content": " "
"type": "Whitespace"
}, },
"pre_tokenizer": null,
"post_processor": { "post_processor": {
"type": "TemplateProcessing", "type": "TemplateProcessing",
"single": [ "single": [
{ "SpecialToken": { "id": "EXAGGERATION", "type_id": 0 } }, {
{ "SpecialToken": { "id": "BOS", "type_id": 0 } }, "SpecialToken": {
{ "Sequence": { "id": "A", "type_id": 0 } }, "id": "[EXAGGERATION]",
{ "SpecialToken": { "id": "EOS", "type_id": 0 } }, "type_id": 0
{ "SpecialToken": { "id": "START_SPEECH", "type_id": 0 } }, }
{ "SpecialToken": { "id": "START_SPEECH", "type_id": 0 } } },
{
"SpecialToken": {
"id": "[START]",
"type_id": 0
}
},
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[STOP]",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[START_SPEECH]",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[START_SPEECH]",
"type_id": 0
}
}
], ],
"pair": [ "pair": [
{ "SpecialToken": { "id": "EXAGGERATION", "type_id": 0 } }, {
{ "SpecialToken": { "id": "BOS", "type_id": 0 } }, "SpecialToken": {
{ "Sequence": { "id": "A", "type_id": 0 } }, "id": "[EXAGGERATION]",
{ "SpecialToken": { "id": "EOS", "type_id": 0 } }, "type_id": 0
{ "SpecialToken": { "id": "START_SPEECH", "type_id": 0 } }, }
{ "SpecialToken": { "id": "START_SPEECH", "type_id": 0 } }, },
{ "SpecialToken": { "id": "EXAGGERATION", "type_id": 1 } }, {
{ "SpecialToken": { "id": "BOS", "type_id": 1 } }, "SpecialToken": {
{ "Sequence": { "id": "B", "type_id": 1 } }, "id": "[START]",
{ "SpecialToken": { "id": "EOS", "type_id": 1 } }, "type_id": 0
{ "SpecialToken": { "id": "START_SPEECH", "type_id": 1 } }, }
{ "SpecialToken": { "id": "START_SPEECH", "type_id": 1 } } },
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[STOP]",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[START_SPEECH]",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[START_SPEECH]",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[EXAGGERATION]",
"type_id": 1
}
},
{
"SpecialToken": {
"id": "[START]",
"type_id": 1
}
},
{
"Sequence": {
"id": "B",
"type_id": 1
}
},
{
"SpecialToken": {
"id": "[STOP]",
"type_id": 1
}
},
{
"SpecialToken": {
"id": "[START_SPEECH]",
"type_id": 1
}
},
{
"SpecialToken": {
"id": "[START_SPEECH]",
"type_id": 1
}
}
], ],
"special_tokens": { "special_tokens": {
"BOS": { "[START]": {
"id": "BOS", "id": "[START]",
"ids": [255], "ids": [
"tokens": ["<s>"] 255
],
"tokens": [
"[START]"
]
}, },
"EOS": { "[STOP]": {
"id": "EOS", "id": "[STOP]",
"ids": [0], "ids": [
"tokens": ["</s>"] 0
],
"tokens": [
"[STOP]"
]
}, },
"EXAGGERATION": { "[EXAGGERATION]": {
"id": "EXAGGERATION", "id": "[EXAGGERATION]",
"ids": [6563], "ids": [
"tokens": ["<EXAGGERATION>"] 6563
],
"tokens": [
"[EXAGGERATION]"
]
}, },
"START_SPEECH": { "[START_SPEECH]": {
"id": "START_SPEECH", "id": "[START_SPEECH]",
"ids": [6561], "ids": [
"tokens": ["<START_SPEECH>"] 6561
],
"tokens": [
"[START_SPEECH]"
]
} }
} }
}, },
"decoder": null, "decoder": {
"type": "Fuse"
},
"model": { "model": {
"type": "BPE", "type": "BPE",
"dropout": null, "dropout": null,
@ -3486,7 +3607,109 @@
"tch": 2348, "tch": 2348,
"sch": 2349, "sch": 2349,
"🙊": 2350, "🙊": 2350,
"🤭": 2351 "🤭": 2351,
"€": 2352,
"أ": 2353,
"إ": 2354,
"ئ": 2355,
"آ": 2356,
"ؤ": 2357,
"ﻻ": 2358,
"ﺃ": 2359,
"ę": 2360,
"ą": 2361,
"ż": 2362,
"ś": 2363,
"ć": 2364,
"ń": 2365,
"ź": 2366,
"Ś": 2367,
"Ź": 2368,
"Ż": 2369,
"Ć": 2370,
"Š": 2371,
"Ő": 2372,
"й": 2373,
"ё": 2374,
"Й": 2375,
"Ё": 2376,
"が": 2377,
"で": 2378,
"じ": 2379,
"だ": 2380,
"ど": 2381,
"ば": 2382,
"げ": 2383,
"ご": 2384,
"ぶ": 2385,
"ぎ": 2386,
"，": 2387,
"（": 2388,
"：": 2389,
"；": 2390,
"？": 2391,
"！": 2392,
"＃": 2393,
" ）": 2394,
"ά": 2395,
"ό": 2396,
"ί": 2397,
"έ": 2398,
"ή": 2399,
"ύ": 2400,
"ώ": 2401,
"Έ": 2402,
"Ό": 2403,
"Ή": 2404,
"ž": 2405,
"š": 2406,
"ū": 2407,
"ş": 2408,
"Ō": 2409,
"ī": 2410,
"č": 2411,
"ř": 2412,
"ă": 2413,
"이": 2414,
"기": 2415,
"요": 2416,
"에": 2417,
"다": 2418,
"을": 2419,
"은": 2420,
"서": 2421,
"니": 2422,
"어": 2423,
"ě": 2424,
"ů": 2425,
"Č": 2426,
"ň": 2427,
"ď": 2428,
"ť": 2429,
"♭": 2430,
"ľ": 2431,
"ĺ": 2432,
"ğ": 2433,
"İ": 2434,
"Ş": 2435,
"ड़": 2436,
"ढ़": 2437,
"ज़": 2438,
"फ़": 2439,
"ख़": 2440,
"क़": 2441,
"ग़": 2442,
"Ά": 2443,
"ϊ": 2444,
"Ί": 2445,
"Ύ": 2446,
"Ώ": 2447,
"ΐ": 2448,
"ϋ": 2449,
"ũ": 2450,
"ụ": 2451,
"ọ": 2452,
"ạ": 2453
}, },
"merges": [ "merges": [
"t h", "t h",