public/model-chatterbox-multilingual-onnx
62
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Upload tokenizer.json

Browse Source
This commit is contained in:
Cherrytest 2025-10-29 03:31:39 +00:00
parent b7051e4619
commit 8bd4b2270f
1 changed files with 104 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
tokenizer.json
View File

@ -3486,7 +3486,109 @@
"tch": 2348, "tch": 2348,
"sch": 2349, "sch": 2349,
"🙊": 2350, "🙊": 2350,
"🤭": 2351 "🤭": 2351,
"€": 2352,
"أ": 2353,
"إ": 2354,
"ئ": 2355,
"آ": 2356,
"ؤ": 2357,
"ﻻ": 2358,
"ﺃ": 2359,
"ę": 2360,
"ą": 2361,
"ż": 2362,
"ś": 2363,
"ć": 2364,
"ń": 2365,
"ź": 2366,
"Ś": 2367,
"Ź": 2368,
"Ż": 2369,
"Ć": 2370,
"Š": 2371,
"Ő": 2372,
"й": 2373,
"ё": 2374,
"Й": 2375,
"Ё": 2376,
"が": 2377,
"で": 2378,
"じ": 2379,
"だ": 2380,
"ど": 2381,
"ば": 2382,
"げ": 2383,
"ご": 2384,
"ぶ": 2385,
"ぎ": 2386,
"，": 2387,
"（": 2388,
"：": 2389,
"；": 2390,
"？": 2391,
"！": 2392,
"＃": 2393,
" ）": 2394,
"ά": 2395,
"ό": 2396,
"ί": 2397,
"έ": 2398,
"ή": 2399,
"ύ": 2400,
"ώ": 2401,
"Έ": 2402,
"Ό": 2403,
"Ή": 2404,
"ž": 2405,
"š": 2406,
"ū": 2407,
"ş": 2408,
"Ō": 2409,
"ī": 2410,
"č": 2411,
"ř": 2412,
"ă": 2413,
"이": 2414,
"기": 2415,
"요": 2416,
"에": 2417,
"다": 2418,
"을": 2419,
"은": 2420,
"서": 2421,
"니": 2422,
"어": 2423,
"ě": 2424,
"ů": 2425,
"Č": 2426,
"ň": 2427,
"ď": 2428,
"ť": 2429,
"♭": 2430,
"ľ": 2431,
"ĺ": 2432,
"ğ": 2433,
"İ": 2434,
"Ş": 2435,
"ड़": 2436,
"ढ़": 2437,
"ज़": 2438,
"फ़": 2439,
"ख़": 2440,
"क़": 2441,
"ग़": 2442,
"Ά": 2443,
"ϊ": 2444,
"Ί": 2445,
"Ύ": 2446,
"Ώ": 2447,
"ΐ": 2448,
"ϋ": 2449,
"ũ": 2450,
"ụ": 2451,
"ọ": 2452,
"ạ": 2453
}, },
"merges": [ "merges": [
"t h", "t h",