Upload tokenizer.json

2025-10-29 03:31:39 +00:00 · 2025-10-29 03:31:39 +00:00 · 8bd4b2270f
commit 8bd4b2270f
parent b7051e4619
1 changed files with 104 additions and 2 deletions
--- a/tokenizer.json
+++ b/tokenizer.json
@ -3486,8 +3486,110 @@
      "tch": 2348,
      "sch": 2349,
      "🙊": 2350,
-      "🤭": 2351
-    },
+      "🤭": 2351,
+      "€": 2352,
+      "أ": 2353,
+      "إ": 2354,
+      "ئ": 2355,
+      "آ": 2356,
+      "ؤ": 2357,
+      "ﻻ": 2358,
+      "ﺃ": 2359,
+      "ę": 2360,
+      "ą": 2361,
+      "ż": 2362,
+      "ś": 2363,
+      "ć": 2364,
+      "ń": 2365,
+      "ź": 2366,
+      "Ś": 2367,
+      "Ź": 2368,
+      "Ż": 2369,
+      "Ć": 2370,
+      "Š": 2371,
+      "Ő": 2372,
+      "й": 2373,
+      "ё": 2374,
+      "Й": 2375,
+      "Ё": 2376,
+      "が": 2377,
+      "で": 2378,
+      "じ": 2379,
+      "だ": 2380,
+      "ど": 2381,
+      "ば": 2382,
+      "げ": 2383,
+      "ご": 2384,
+      "ぶ": 2385,
+      "ぎ": 2386,
+      "，": 2387,
+      "（": 2388,
+      "：": 2389,
+      "；": 2390,
+      "？": 2391,
+      "！": 2392,
+      "＃": 2393,
+      " ）": 2394,
+      "ά": 2395,
+      "ό": 2396,
+      "ί": 2397,
+      "έ": 2398,
+      "ή": 2399,
+      "ύ": 2400,
+      "ώ": 2401,
+      "Έ": 2402,
+      "Ό": 2403,
+      "Ή": 2404, 
+      "ž": 2405,
+      "š": 2406,
+      "ū": 2407,
+      "ş": 2408,
+      "Ō": 2409,
+      "ī": 2410,
+      "č": 2411,
+      "ř": 2412,
+      "ă": 2413,
+      "이": 2414,
+      "기": 2415,
+      "요": 2416,
+      "에": 2417,
+      "다": 2418,
+      "을": 2419,
+      "은": 2420,
+      "서": 2421,
+      "니": 2422,
+      "어": 2423,
+      "ě": 2424,
+      "ů": 2425,
+      "Č": 2426,
+      "ň": 2427,
+      "ď": 2428,
+      "ť": 2429,
+      "♭": 2430,
+      "ľ": 2431,
+      "ĺ": 2432,
+      "ğ": 2433,
+      "İ": 2434,
+      "Ş": 2435,
+      "ड़": 2436,
+      "ढ़": 2437,
+      "ज़": 2438,
+      "फ़": 2439,
+      "ख़": 2440,
+      "क़": 2441,
+      "ग़": 2442,
+      "Ά": 2443,
+      "ϊ": 2444,
+      "Ί": 2445,
+      "Ύ": 2446,
+      "Ώ": 2447,
+      "ΐ": 2448,
+      "ϋ": 2449,
+      "ũ": 2450,
+      "ụ": 2451,
+      "ọ": 2452,
+      "ạ": 2453
+          },
    "merges": [
      "t h",
      "i n",