public/model-chatterbox-multilingual-onnx
62
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Upload tokenizer.json

Browse Source
This commit is contained in:
Cherrytest 2025-09-30 03:55:57 +00:00
parent 985eb49a65
commit 97323c8c35
1 changed files with 280 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

337
tokenizer.json
View File

@ -21,15 +21,6 @@
"rstrip": false,
"normalized": false
},
{
"id": 2,
"special": true,
"content": "[SPACE]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true
},
{
"id": 255,
"special": true,
@ -1064,69 +1055,199 @@
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 6561,
"content": "[START_SPEECH]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 6562,
"content": "[STOP_SPEECH]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 6563,
"content": "[EXAGGERATION]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": {
"type": "Sequence",
"normalizers": [
{
"type": "Replace",
"pattern": { "String": " " },
"content": "[SPACE]"
}
]
},
"pre_tokenizer": {
"type": "Whitespace"
"type": "Replace",
"pattern": {
"Regex": "\\s+"
},
"content": " "
},
"pre_tokenizer": null,
"post_processor": {
"type": "TemplateProcessing",
"single": [
{ "SpecialToken": { "id": "EXAGGERATION", "type_id": 0 } },
{ "SpecialToken": { "id": "BOS", "type_id": 0 } },
{ "Sequence": { "id": "A", "type_id": 0 } },
{ "SpecialToken": { "id": "EOS", "type_id": 0 } },
{ "SpecialToken": { "id": "START_SPEECH", "type_id": 0 } },
{ "SpecialToken": { "id": "START_SPEECH", "type_id": 0 } }
{
"SpecialToken": {
"id": "[EXAGGERATION]",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[START]",
"type_id": 0
}
},
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[STOP]",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[START_SPEECH]",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[START_SPEECH]",
"type_id": 0
}
}
],
"pair": [
{ "SpecialToken": { "id": "EXAGGERATION", "type_id": 0 } },
{ "SpecialToken": { "id": "BOS", "type_id": 0 } },
{ "Sequence": { "id": "A", "type_id": 0 } },
{ "SpecialToken": { "id": "EOS", "type_id": 0 } },
{ "SpecialToken": { "id": "START_SPEECH", "type_id": 0 } },
{ "SpecialToken": { "id": "START_SPEECH", "type_id": 0 } },
{ "SpecialToken": { "id": "EXAGGERATION", "type_id": 1 } },
{ "SpecialToken": { "id": "BOS", "type_id": 1 } },
{ "Sequence": { "id": "B", "type_id": 1 } },
{ "SpecialToken": { "id": "EOS", "type_id": 1 } },
{ "SpecialToken": { "id": "START_SPEECH", "type_id": 1 } },
{ "SpecialToken": { "id": "START_SPEECH", "type_id": 1 } }
{
"SpecialToken": {
"id": "[EXAGGERATION]",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[START]",
"type_id": 0
}
},
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[STOP]",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[START_SPEECH]",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[START_SPEECH]",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[EXAGGERATION]",
"type_id": 1
}
},
{
"SpecialToken": {
"id": "[START]",
"type_id": 1
}
},
{
"Sequence": {
"id": "B",
"type_id": 1
}
},
{
"SpecialToken": {
"id": "[STOP]",
"type_id": 1
}
},
{
"SpecialToken": {
"id": "[START_SPEECH]",
"type_id": 1
}
},
{
"SpecialToken": {
"id": "[START_SPEECH]",
"type_id": 1
}
}
],
"special_tokens": {
"BOS": {
"id": "BOS",
"ids": [255],
"tokens": ["<s>"]
"[START]": {
"id": "[START]",
"ids": [
255
],
"tokens": [
"[START]"
]
},
"EOS": {
"id": "EOS",
"ids": [0],
"tokens": ["</s>"]
"[STOP]": {
"id": "[STOP]",
"ids": [
0
],
"tokens": [
"[STOP]"
]
},
"EXAGGERATION": {
"id": "EXAGGERATION",
"ids": [6563],
"tokens": ["<EXAGGERATION>"]
"[EXAGGERATION]": {
"id": "[EXAGGERATION]",
"ids": [
6563
],
"tokens": [
"[EXAGGERATION]"
]
},
"START_SPEECH": {
"id": "START_SPEECH",
"ids": [6561],
"tokens": ["<START_SPEECH>"]
"[START_SPEECH]": {
"id": "[START_SPEECH]",
"ids": [
6561
],
"tokens": [
"[START_SPEECH]"
]
}
}
},
"decoder": null,
"decoder": {
"type": "Fuse"
},
"model": {
"type": "BPE",
"dropout": null,
@ -3486,8 +3607,110 @@
"tch": 2348,
"sch": 2349,
"🙊": 2350,
"🤭": 2351
},
"🤭": 2351,
"€": 2352,
"أ": 2353,
"إ": 2354,
"ئ": 2355,
"آ": 2356,
"ؤ": 2357,
"ﻻ": 2358,
"ﺃ": 2359,
"ę": 2360,
"ą": 2361,
"ż": 2362,
"ś": 2363,
"ć": 2364,
"ń": 2365,
"ź": 2366,
"Ś": 2367,
"Ź": 2368,
"Ż": 2369,
"Ć": 2370,
"Š": 2371,
"Ő": 2372,
"й": 2373,
"ё": 2374,
"Й": 2375,
"Ё": 2376,
"が": 2377,
"で": 2378,
"じ": 2379,
"だ": 2380,
"ど": 2381,
"ば": 2382,
"げ": 2383,
"ご": 2384,
"ぶ": 2385,
"ぎ": 2386,
"，": 2387,
"（": 2388,
"：": 2389,
"；": 2390,
"？": 2391,
"！": 2392,
"＃": 2393,
" ）": 2394,
"ά": 2395,
"ό": 2396,
"ί": 2397,
"έ": 2398,
"ή": 2399,
"ύ": 2400,
"ώ": 2401,
"Έ": 2402,
"Ό": 2403,
"Ή": 2404,
"ž": 2405,
"š": 2406,
"ū": 2407,
"ş": 2408,
"Ō": 2409,
"ī": 2410,
"č": 2411,
"ř": 2412,
"ă": 2413,
"이": 2414,
"기": 2415,
"요": 2416,
"에": 2417,
"다": 2418,
"을": 2419,
"은": 2420,
"서": 2421,
"니": 2422,
"어": 2423,
"ě": 2424,
"ů": 2425,
"Č": 2426,
"ň": 2427,
"ď": 2428,
"ť": 2429,
"♭": 2430,
"ľ": 2431,
"ĺ": 2432,
"ğ": 2433,
"İ": 2434,
"Ş": 2435,
"ड़": 2436,
"ढ़": 2437,
"ज़": 2438,
"फ़": 2439,
"ख़": 2440,
"क़": 2441,
"ग़": 2442,
"Ά": 2443,
"ϊ": 2444,
"Ί": 2445,
"Ύ": 2446,
"Ώ": 2447,
"ΐ": 2448,
"ϋ": 2449,
"ũ": 2450,
"ụ": 2451,
"ọ": 2452,
"ạ": 2453
},
"merges": [
"t h",
"i n",