{"_id":"@cyberlangke/tokkit-core","_rev":"14-f510e9fd8b4a30d09277edb38f913d3a","name":"@cyberlangke/tokkit-core","dist-tags":{"latest":"1.11.0"},"versions":{"1.1.0":{"name":"@cyberlangke/tokkit-core","version":"1.1.0","author":{"name":"cyberlangke"},"license":"MIT","_id":"@cyberlangke/tokkit-core@1.1.0","maintainers":[{"name":"cyberlangke","email":"3531702083@qq.com"}],"homepage":"https://github.com/1cyberlangke1/tokkit#readme","bugs":{"url":"https://github.com/1cyberlangke1/tokkit/issues"},"dist":{"shasum":"0008818aa0e40449388ba19c3c7adf0ffef5ffab","tarball":"https://registry.npmjs.org/@cyberlangke/tokkit-core/-/tokkit-core-1.1.0.tgz","fileCount":28,"integrity":"sha512-ms2HWadDE58JPGyaYcVLUWaV2gMPQJgA4WY5W6fSEjzO778P9DneildwZukzY5bZrOjAokYakB+GkAe7n2sjFQ==","signatures":[{"sig":"MEUCIBqRWarMrsYXsz2ox3QWcAVr/6FS0qI71rBw0wYIHQRAAiEAgvCAkz5L+StmqoW/BNiMCTwk+luNoTjNq2qbGu9cuRA=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":121481},"main":"./dist/index.cjs","type":"module","types":"./dist/index.d.ts","module":"./dist/index.js","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","require":"./dist/index.cjs"}},"gitHead":"999ef11b8f939a6bc82ca8b5388db0900b80a96d","scripts":{"dev":"tsup --watch","build":"tsup"},"_npmUser":{"name":"cyberlangke","email":"3531702083@qq.com"},"repository":{"url":"git+https://github.com/1cyberlangke1/tokkit.git","type":"git"},"_npmVersion":"11.6.0","description":"Core runtime for counting BPE tokens.","directories":{},"_nodeVersion":"24.9.0","publishConfig":{"access":"public"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/tokkit-core_1.1.0_1774661290949_0.7582913292700355","host":"s3://npm-registry-packages-npm-production"}},"1.2.1":{"name":"@cyberlangke/tokkit-core","version":"1.2.1","author":{"name":"cyberlangke"},"license":"MIT","_id":"@cyberlangke/tokkit-core@1.2.1","maintainers":[{"name":"cyberlangke","email":"3531702083@qq.com"}],"homepage":"https://github.com/1cyberlangke1/tokkit#readme","bugs":{"url":"https://github.com/1cyberlangke1/tokkit/issues"},"dist":{"shasum":"ada2e99568bfdbada7e18662c0a0afe012327c61","tarball":"https://registry.npmjs.org/@cyberlangke/tokkit-core/-/tokkit-core-1.2.1.tgz","fileCount":30,"integrity":"sha512-BmpeOSKhfVpK68+VhLY9KIONghrnep/Ry+0sNQlbjbcN2i8mZhITq4dbhvkqKm62GBs7P9JHFnMJmnjBXol2fg==","signatures":[{"sig":"MEYCIQCsgfs2YmpZcquhJ/wyeY0bERMhGZQmolJunXXFYhbisQIhALuya0/N0pAOGfmxlEJ0+E3a+24R+LfJgAqi9uM90xaE","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":130532},"main":"./dist/index.cjs","type":"module","types":"./dist/index.d.ts","module":"./dist/index.js","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","require":"./dist/index.cjs"}},"gitHead":"4c6d4f8a7edcbebfd58cdf5301eb675242f4bb21","scripts":{"dev":"tsup --watch","build":"tsup"},"_npmUser":{"name":"cyberlangke","email":"3531702083@qq.com"},"repository":{"url":"git+https://github.com/1cyberlangke1/tokkit.git","type":"git"},"_npmVersion":"11.9.0","description":"Core runtime for counting BPE tokens.","directories":{},"_nodeVersion":"24.14.0","publishConfig":{"access":"public"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/tokkit-core_1.2.1_1774688729068_0.40467475600356884","host":"s3://npm-registry-packages-npm-production"}},"1.2.2":{"name":"@cyberlangke/tokkit-core","version":"1.2.2","author":{"name":"cyberlangke"},"license":"MIT","_id":"@cyberlangke/tokkit-core@1.2.2","maintainers":[{"name":"cyberlangke","email":"3531702083@qq.com"}],"homepage":"https://github.com/1cyberlangke1/tokkit#readme","bugs":{"url":"https://github.com/1cyberlangke1/tokkit/issues"},"dist":{"shasum":"67e66889e21df16f08d18125984299d91ff28368","tarball":"https://registry.npmjs.org/@cyberlangke/tokkit-core/-/tokkit-core-1.2.2.tgz","fileCount":30,"integrity":"sha512-bfBfNDegbpOYJTMfEGrUXjuFOb6G0SITrvGBNJbRid4kLcBjp3BE+7oKVfXYY97SvY8GfqBZtqJ3Pt3+a55BPA==","signatures":[{"sig":"MEUCIG/bO1o53A28X4uAdRHGBMbURqMi544hAs1BeK9b9g17AiEAx927KWD7RzFBRaT98/38kryefDooqSmVLBG6pzMA61k=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":132134},"main":"./dist/index.cjs","type":"module","types":"./dist/index.d.ts","module":"./dist/index.js","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","require":"./dist/index.cjs"}},"gitHead":"0853008708fc4912c226f8f4810a3c84ce86ac26","scripts":{"dev":"tsup --watch","build":"tsup"},"_npmUser":{"name":"cyberlangke","email":"3531702083@qq.com"},"repository":{"url":"git+https://github.com/1cyberlangke1/tokkit.git","type":"git"},"_npmVersion":"11.9.0","description":"Core runtime for counting BPE tokens.","directories":{},"_nodeVersion":"24.14.0","publishConfig":{"access":"public"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/tokkit-core_1.2.2_1774689983521_0.9421966169518936","host":"s3://npm-registry-packages-npm-production"}},"1.2.3":{"name":"@cyberlangke/tokkit-core","version":"1.2.3","author":{"name":"cyberlangke"},"license":"MIT","_id":"@cyberlangke/tokkit-core@1.2.3","maintainers":[{"name":"cyberlangke","email":"3531702083@qq.com"}],"homepage":"https://github.com/1cyberlangke1/tokkit#readme","bugs":{"url":"https://github.com/1cyberlangke1/tokkit/issues"},"dist":{"shasum":"c0f17e8daeb588e94bbccd1e2bae47bde2b3fa3b","tarball":"https://registry.npmjs.org/@cyberlangke/tokkit-core/-/tokkit-core-1.2.3.tgz","fileCount":30,"integrity":"sha512-jjGBd5WBlrp0VI8jBtAYXnVAmS+zWMWoXQCwTwxBNC9yW7UacM40HeGmVpyTDutvvOQUvLQ3vFMOUnDVR4ETDA==","signatures":[{"sig":"MEUCIHExeI2DNt/laH/EnnUZPtc7SYF1xoLt94XaBbFHCCZeAiEA6RJbtwI8vmhgPfi24A7rgMebn1V9L3B/Li6uWTe4lsU=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":134466},"main":"./dist/index.cjs","type":"module","types":"./dist/index.d.ts","module":"./dist/index.js","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","require":"./dist/index.cjs"}},"gitHead":"9a55c7924d481a97a9059aeb9b31ec6d8c7aaa95","scripts":{"dev":"tsup --watch","build":"tsup"},"_npmUser":{"name":"cyberlangke","email":"3531702083@qq.com"},"repository":{"url":"git+https://github.com/1cyberlangke1/tokkit.git","type":"git"},"_npmVersion":"11.9.0","description":"Core runtime for counting BPE tokens.","directories":{},"_nodeVersion":"24.14.0","publishConfig":{"access":"public"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/tokkit-core_1.2.3_1774691436300_0.46106927459861313","host":"s3://npm-registry-packages-npm-production"}},"1.3.0":{"name":"@cyberlangke/tokkit-core","version":"1.3.0","author":{"name":"cyberlangke"},"license":"MIT","_id":"@cyberlangke/tokkit-core@1.3.0","maintainers":[{"name":"cyberlangke","email":"3531702083@qq.com"}],"homepage":"https://github.com/1cyberlangke1/tokkit#readme","bugs":{"url":"https://github.com/1cyberlangke1/tokkit/issues"},"dist":{"shasum":"3cfceaa9e7fe1cef08f1e2bd0f5c76ed83fc260f","tarball":"https://registry.npmjs.org/@cyberlangke/tokkit-core/-/tokkit-core-1.3.0.tgz","fileCount":30,"integrity":"sha512-IGmco1oWxsFkZ6l+aLqLhLvX/4Nqtm2N5fGDXOtM7PEbRIdoL7zlTx052SLZndWq2s7QHbss2JLvRKRtkiOWAQ==","signatures":[{"sig":"MEUCICa/9FtUE/vFRFCCMHQ2mZLMpx+pnNFsDnqdIu3Hp//IAiEAqUe19joBfZwzk1t456mNvU3GXESIcDBoWlCte8tEkOY=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":134466},"main":"./dist/index.cjs","type":"module","types":"./dist/index.d.ts","module":"./dist/index.js","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","require":"./dist/index.cjs"}},"gitHead":"8a2217da552ba62538637e780157ae780baf6228","scripts":{"dev":"tsup --watch","build":"tsup"},"_npmUser":{"name":"cyberlangke","email":"3531702083@qq.com"},"repository":{"url":"git+https://github.com/1cyberlangke1/tokkit.git","type":"git"},"_npmVersion":"11.9.0","description":"Core runtime for counting BPE tokens.","directories":{},"_nodeVersion":"24.14.0","publishConfig":{"access":"public"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/tokkit-core_1.3.0_1774694512224_0.006619107765765797","host":"s3://npm-registry-packages-npm-production"}},"1.4.0":{"name":"@cyberlangke/tokkit-core","version":"1.4.0","author":{"name":"cyberlangke"},"license":"MIT","_id":"@cyberlangke/tokkit-core@1.4.0","maintainers":[{"name":"cyberlangke","email":"3531702083@qq.com"}],"homepage":"https://github.com/1cyberlangke1/tokkit#readme","bugs":{"url":"https://github.com/1cyberlangke1/tokkit/issues"},"dist":{"shasum":"fdb2b66a430811e63073819c0067bcab650b23d4","tarball":"https://registry.npmjs.org/@cyberlangke/tokkit-core/-/tokkit-core-1.4.0.tgz","fileCount":30,"integrity":"sha512-znxC5qi19WlHMUpNOYEGV0vmJMQJa0MG9YvNDU0YaAyyEpeLkhSRpuUIXdyndOxSvAn8marX1AOZQ8ZMJInhgA==","signatures":[{"sig":"MEQCIFct3oSR3YFJX5UEELVgnr6yXMeRsc1Q7KKRqE6e3/XzAiBEe0N9Q7TR9n37xXbq/HNK6PypgKSc3M55nd+O+qWc0w==","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":134466},"main":"./dist/index.cjs","type":"module","types":"./dist/index.d.ts","module":"./dist/index.js","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","require":"./dist/index.cjs"}},"gitHead":"4956169455ee4dd6aa70086d50da9d79ecb4232d","scripts":{"dev":"tsup --watch","build":"tsup"},"_npmUser":{"name":"cyberlangke","email":"3531702083@qq.com"},"repository":{"url":"git+https://github.com/1cyberlangke1/tokkit.git","type":"git"},"_npmVersion":"11.9.0","description":"Core runtime for counting BPE tokens.","directories":{},"_nodeVersion":"24.14.0","publishConfig":{"access":"public"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/tokkit-core_1.4.0_1774697326615_0.6572170398611503","host":"s3://npm-registry-packages-npm-production"}},"1.4.1":{"name":"@cyberlangke/tokkit-core","version":"1.4.1","author":{"name":"cyberlangke"},"license":"MIT","_id":"@cyberlangke/tokkit-core@1.4.1","maintainers":[{"name":"cyberlangke","email":"3531702083@qq.com"}],"homepage":"https://github.com/1cyberlangke1/tokkit#readme","bugs":{"url":"https://github.com/1cyberlangke1/tokkit/issues"},"dist":{"shasum":"e68d1a70ccb10d0e3ef71d8276699b5907888523","tarball":"https://registry.npmjs.org/@cyberlangke/tokkit-core/-/tokkit-core-1.4.1.tgz","fileCount":30,"integrity":"sha512-VxTwzjv04jDuH/dBHW5l/2T2wGW8lThby5YxC4+hCiqGjiMI05k41BvbRAodYFYIHgwRBUTI+b90EA461fKJFw==","signatures":[{"sig":"MEQCIF7C6dBoZFn+t6xYXVG87Qunu1OdHMO0IvHzC679lWAJAiBDxIwHwe7HS8eOHvmLD/i4Dvyn3xvH4iNsfxVvtaBwaA==","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":143154},"main":"./dist/index.cjs","type":"module","types":"./dist/index.d.ts","module":"./dist/index.js","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","require":"./dist/index.cjs"}},"gitHead":"a6dd2eb8a3c18dae80561a28114c73ed73ffbcc1","scripts":{"dev":"tsup --watch","build":"tsup"},"_npmUser":{"name":"cyberlangke","email":"3531702083@qq.com"},"repository":{"url":"git+https://github.com/1cyberlangke1/tokkit.git","type":"git"},"_npmVersion":"11.9.0","description":"Core runtime for counting BPE tokens.","directories":{},"_nodeVersion":"24.14.0","publishConfig":{"access":"public"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/tokkit-core_1.4.1_1774766102901_0.9035718668142405","host":"s3://npm-registry-packages-npm-production"}},"1.5.0":{"name":"@cyberlangke/tokkit-core","version":"1.5.0","author":{"name":"cyberlangke"},"license":"MIT","_id":"@cyberlangke/tokkit-core@1.5.0","maintainers":[{"name":"cyberlangke","email":"3531702083@qq.com"}],"homepage":"https://github.com/1cyberlangke1/tokkit#readme","bugs":{"url":"https://github.com/1cyberlangke1/tokkit/issues"},"dist":{"shasum":"644040d9d2597e1e4d10246c7122d588c4b0ea2c","tarball":"https://registry.npmjs.org/@cyberlangke/tokkit-core/-/tokkit-core-1.5.0.tgz","fileCount":30,"integrity":"sha512-LkGpVod8bywgYoQln6Yh7JSPdqR80eShyLPLFVES8oXkFjV0QCoCp+Bi3A9R5ONOOIScgdqbv8QTmu1Ya9JR9A==","signatures":[{"sig":"MEUCIHOb2f+6DQ8Fr5BIitAMDHSlFN97uzRcmmOu4tIWb7w4AiEAw0437ve2sGVxkKKazNwWkrXXTyD9O7LkN7dpyN49Log=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":143154},"main":"./dist/index.cjs","type":"module","types":"./dist/index.d.ts","module":"./dist/index.js","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","require":"./dist/index.cjs"}},"gitHead":"d6c68540588c0952d2f3d26b9a3a8f96f4069384","scripts":{"dev":"tsup --watch","build":"tsup"},"_npmUser":{"name":"cyberlangke","email":"3531702083@qq.com"},"repository":{"url":"git+https://github.com/1cyberlangke1/tokkit.git","type":"git"},"_npmVersion":"11.9.0","description":"Core runtime for counting BPE tokens.","directories":{},"_nodeVersion":"24.14.0","publishConfig":{"access":"public"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/tokkit-core_1.5.0_1774853677869_0.06610498766167083","host":"s3://npm-registry-packages-npm-production"}},"1.6.0":{"name":"@cyberlangke/tokkit-core","version":"1.6.0","author":{"name":"cyberlangke"},"license":"MIT","_id":"@cyberlangke/tokkit-core@1.6.0","maintainers":[{"name":"cyberlangke","email":"3531702083@qq.com"}],"homepage":"https://github.com/1cyberlangke1/tokkit#readme","bugs":{"url":"https://github.com/1cyberlangke1/tokkit/issues"},"dist":{"shasum":"57a9c4ae739d9a29f37b2ed47602a09eb213b1c5","tarball":"https://registry.npmjs.org/@cyberlangke/tokkit-core/-/tokkit-core-1.6.0.tgz","fileCount":30,"integrity":"sha512-igzprvF3UTE3L+93R4b3MUrxdHJ4nPXQgiI765ZhFYIw/YvoYQl3uluQjoNxW2HQqaLnTLGSusQuhw08AXvDJg==","signatures":[{"sig":"MEUCICUqPzgqO2iJfyjTvdSmahS96xExnxnbAOkbW2ftjHBkAiEAz370faAOgXy6ygLOaPvIhhLipY0UgPGC4Tckli3hJUk=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":143154},"main":"./dist/index.cjs","type":"module","types":"./dist/index.d.ts","module":"./dist/index.js","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","require":"./dist/index.cjs"}},"gitHead":"f5596e039fcd805883f5c0cdecf7ab6cc8ce0015","scripts":{"dev":"tsup --watch","build":"tsup"},"_npmUser":{"name":"cyberlangke","email":"3531702083@qq.com"},"repository":{"url":"git+https://github.com/1cyberlangke1/tokkit.git","type":"git"},"_npmVersion":"11.9.0","description":"Core runtime for counting BPE tokens.","directories":{},"_nodeVersion":"24.14.0","publishConfig":{"access":"public"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/tokkit-core_1.6.0_1774861758727_0.10536809442377981","host":"s3://npm-registry-packages-npm-production"}},"1.7.0":{"name":"@cyberlangke/tokkit-core","version":"1.7.0","author":{"name":"cyberlangke"},"license":"MIT","_id":"@cyberlangke/tokkit-core@1.7.0","maintainers":[{"name":"cyberlangke","email":"3531702083@qq.com"}],"homepage":"https://github.com/1cyberlangke1/tokkit#readme","bugs":{"url":"https://github.com/1cyberlangke1/tokkit/issues"},"dist":{"shasum":"c8902cefd78a9e4229ee2b1151518ef683f62fd8","tarball":"https://registry.npmjs.org/@cyberlangke/tokkit-core/-/tokkit-core-1.7.0.tgz","fileCount":30,"integrity":"sha512-958xDT8EsDmZTmIH4L9N/hpGq/5o1Jw++M0biSYPgy1ldq9T6/Iy7P0BkC4aqgxGfv3SVcs4QLUWOod7dKG29Q==","signatures":[{"sig":"MEQCIBnddz54Z4wtpWWv/dBxwwnOEc3W/Mw0J3mBv6JCibjmAiAVB/n3eabn+lQHkLLXT3a66k0Y6uqxzlEjwMKPbd6/rA==","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":143154},"main":"./dist/index.cjs","type":"module","types":"./dist/index.d.ts","module":"./dist/index.js","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","require":"./dist/index.cjs"}},"gitHead":"d30f8c18c181d49654ac20cb0bf2f47eb012cabb","scripts":{"dev":"tsup --watch","build":"tsup"},"_npmUser":{"name":"cyberlangke","email":"3531702083@qq.com"},"repository":{"url":"git+https://github.com/1cyberlangke1/tokkit.git","type":"git"},"_npmVersion":"11.9.0","description":"Core runtime for counting BPE tokens.","directories":{},"_nodeVersion":"24.14.0","publishConfig":{"access":"public"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/tokkit-core_1.7.0_1774866236001_0.11237935561678891","host":"s3://npm-registry-packages-npm-production"}},"1.8.0":{"name":"@cyberlangke/tokkit-core","version":"1.8.0","author":{"name":"cyberlangke"},"license":"MIT","_id":"@cyberlangke/tokkit-core@1.8.0","maintainers":[{"name":"cyberlangke","email":"3531702083@qq.com"}],"homepage":"https://github.com/1cyberlangke1/tokkit#readme","bugs":{"url":"https://github.com/1cyberlangke1/tokkit/issues"},"dist":{"shasum":"0490db1a43a5ceb885b31878dd427aabdc06620c","tarball":"https://registry.npmjs.org/@cyberlangke/tokkit-core/-/tokkit-core-1.8.0.tgz","fileCount":30,"integrity":"sha512-6j/f7AUx+iuHc/mXxnqG3hWFGyIsNCNvYlQRI8kj6YmM/5PSoONZIFY6kAiqHUCakbhX8LFbQJpMCGjhoXXcdQ==","signatures":[{"sig":"MEUCIQCZsjGk94F5ycmHuzZkOE9s3sVm7GabNAXxbLfIjlxGSAIgfH9Vtu7gdgA/3AZgl4xNlnKGbEN9ObDeP5E8KyqcgnE=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":143154},"main":"./dist/index.cjs","type":"module","types":"./dist/index.d.ts","module":"./dist/index.js","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","require":"./dist/index.cjs"}},"gitHead":"7e5f33d0fbdb6c8184701b624b61652bea7f8a99","scripts":{"dev":"tsup --watch","build":"tsup"},"_npmUser":{"name":"cyberlangke","email":"3531702083@qq.com"},"repository":{"url":"git+https://github.com/1cyberlangke1/tokkit.git","type":"git"},"_npmVersion":"11.9.0","description":"Core runtime for counting BPE tokens.","directories":{},"_nodeVersion":"24.14.0","publishConfig":{"access":"public"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/tokkit-core_1.8.0_1774869317090_0.8865587942729407","host":"s3://npm-registry-packages-npm-production"}},"1.9.0":{"name":"@cyberlangke/tokkit-core","version":"1.9.0","author":{"name":"cyberlangke"},"license":"MIT","_id":"@cyberlangke/tokkit-core@1.9.0","maintainers":[{"name":"cyberlangke","email":"3531702083@qq.com"}],"homepage":"https://github.com/1cyberlangke1/tokkit#readme","bugs":{"url":"https://github.com/1cyberlangke1/tokkit/issues"},"dist":{"shasum":"31cde623d122800cacbb941422c5339858ca1393","tarball":"https://registry.npmjs.org/@cyberlangke/tokkit-core/-/tokkit-core-1.9.0.tgz","fileCount":30,"integrity":"sha512-sfqaSfHPryZ0qEPmVyRERX6r86iR2qUJ7CPUnmT1gsdH/ijJlZ033TZLNCr4L+CYce6hRohhcGgg/9Mor84nbQ==","signatures":[{"sig":"MEUCIQCZdAIZqRPDs/PnHs1bDjT8szyjW86Y6IGfeRjGKMR7+QIgOUhtsH/GLZHvZStMoRB0dyW0yKGsOkqfU//RF2ajUXg=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":143154},"main":"./dist/index.cjs","type":"module","types":"./dist/index.d.ts","module":"./dist/index.js","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","require":"./dist/index.cjs"}},"gitHead":"966b217b2c25444ce482773a6fedf7b5b9d24608","scripts":{"dev":"tsup --watch","build":"tsup"},"_npmUser":{"name":"cyberlangke","email":"3531702083@qq.com"},"repository":{"url":"git+https://github.com/1cyberlangke1/tokkit.git","type":"git"},"_npmVersion":"11.9.0","description":"Core runtime for counting BPE tokens.","directories":{},"_nodeVersion":"24.14.0","publishConfig":{"access":"public"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/tokkit-core_1.9.0_1774872835987_0.7742267846619801","host":"s3://npm-registry-packages-npm-production"}},"1.10.0":{"name":"@cyberlangke/tokkit-core","version":"1.10.0","author":{"name":"cyberlangke"},"license":"MIT","_id":"@cyberlangke/tokkit-core@1.10.0","maintainers":[{"name":"cyberlangke","email":"3531702083@qq.com"}],"homepage":"https://github.com/1cyberlangke1/tokkit#readme","bugs":{"url":"https://github.com/1cyberlangke1/tokkit/issues"},"dist":{"shasum":"b92b1f2f6f59e4bdfe4ff08cbe8fb4108559971b","tarball":"https://registry.npmjs.org/@cyberlangke/tokkit-core/-/tokkit-core-1.10.0.tgz","fileCount":30,"integrity":"sha512-HCwLziL3SBThF8m5STCqD+WiC80jXe4c6H42wB0dXoReQv7BRfW6V1AhxF2JHXu+XBOq4PbcGRHBp0xBqBKoxA==","signatures":[{"sig":"MEUCIHyQcLOYTlMRcIWXrfSxXtXv+X+XBqO54cB99nxypMQYAiEA3+R5bZiCWGUjwdE9K5yWPvxOFPYVQDw2gzodN2Morjw=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":143155},"main":"./dist/index.cjs","type":"module","types":"./dist/index.d.ts","module":"./dist/index.js","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","require":"./dist/index.cjs"}},"gitHead":"c086a200b2f732709de752de4b7120a28d55cfd2","scripts":{"dev":"tsup --watch","build":"tsup"},"_npmUser":{"name":"cyberlangke","email":"3531702083@qq.com"},"repository":{"url":"git+https://github.com/1cyberlangke1/tokkit.git","type":"git"},"_npmVersion":"11.9.0","description":"Core runtime for counting BPE tokens.","directories":{},"_nodeVersion":"24.14.0","publishConfig":{"access":"public"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/tokkit-core_1.10.0_1774875393324_0.20509904379223998","host":"s3://npm-registry-packages-npm-production"}},"1.11.0":{"name":"@cyberlangke/tokkit-core","version":"1.11.0","description":"Core runtime for counting BPE tokens.","type":"module","main":"./dist/index.cjs","module":"./dist/index.js","types":"./dist/index.d.ts","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","require":"./dist/index.cjs"}},"scripts":{"build":"tsup","dev":"tsup --watch"},"publishConfig":{"access":"public"},"repository":{"type":"git","url":"git+https://github.com/1cyberlangke1/tokkit.git"},"homepage":"https://github.com/1cyberlangke1/tokkit#readme","bugs":{"url":"https://github.com/1cyberlangke1/tokkit/issues"},"author":{"name":"cyberlangke"},"license":"MIT","gitHead":"6d21c3bb1df377d5de599859bf065685223c2dc3","_id":"@cyberlangke/tokkit-core@1.11.0","_nodeVersion":"24.14.0","_npmVersion":"11.9.0","dist":{"integrity":"sha512-ku2CEs/7cogb/2OcTgR1Y/qM+ugT2jJhbd8Dq2rbT+lQ5XK0g5mEyXIkZJgl97QQqwDtRhp5iXqk1wMXo2PZLw==","shasum":"3f3d369c50522a1b660f69ec4d78df2440b719dc","tarball":"https://registry.npmjs.org/@cyberlangke/tokkit-core/-/tokkit-core-1.11.0.tgz","fileCount":30,"unpackedSize":143155,"signatures":[{"keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U","sig":"MEYCIQDIFplgY7kFzEkgLb+jsZacoxSTfmw6srTaeMMXQ4tJ5wIhALW8+PSbfakIPNeuDepcWlZf21Ob3mST3iborPCDN6ng"}]},"_npmUser":{"name":"cyberlangke","email":"3531702083@qq.com"},"directories":{},"maintainers":[{"name":"cyberlangke","email":"3531702083@qq.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages-npm-production","tmp":"tmp/tokkit-core_1.11.0_1774877962286_0.1396479314271688"},"_hasShrinkwrap":false}},"time":{"created":"2026-03-28T01:28:10.873Z","modified":"2026-03-30T13:39:22.806Z","1.1.0":"2026-03-28T01:28:11.104Z","1.2.1":"2026-03-28T09:05:29.243Z","1.2.2":"2026-03-28T09:26:23.651Z","1.2.3":"2026-03-28T09:50:36.437Z","1.3.0":"2026-03-28T10:41:52.377Z","1.4.0":"2026-03-28T11:28:46.782Z","1.4.1":"2026-03-29T06:35:03.055Z","1.5.0":"2026-03-30T06:54:38.019Z","1.6.0":"2026-03-30T09:09:19.664Z","1.7.0":"2026-03-30T10:23:56.151Z","1.8.0":"2026-03-30T11:15:17.226Z","1.9.0":"2026-03-30T12:13:56.133Z","1.10.0":"2026-03-30T12:56:33.500Z","1.11.0":"2026-03-30T13:39:22.424Z"},"bugs":{"url":"https://github.com/1cyberlangke1/tokkit/issues"},"author":{"name":"cyberlangke"},"license":"MIT","homepage":"https://github.com/1cyberlangke1/tokkit#readme","repository":{"type":"git","url":"git+https://github.com/1cyberlangke1/tokkit.git"},"description":"Core runtime for counting BPE tokens.","maintainers":[{"name":"cyberlangke","email":"3531702083@qq.com"}],"readme":"# @cyberlangke/tokkit-core\n\nTokkit 的核心运行时包，提供 BPE 分词、反分词、注册表与自定义 family 注册能力。\n\n## 使用方法\n\n```bash\nnpm install @cyberlangke/tokkit-core\n```\n\n```ts\nimport { registerTokenizerFamily, getTokenizer } from \"@cyberlangke/tokkit-core\"\n\n// 注册自定义 family 定义。\nregisterTokenizerFamily({\n  family: \"toy\",\n  asset: {\n    added_tokens: [],\n    model: {\n      type: \"BPE\",\n      vocab: { a: 0 },\n      merges: [],\n      unk_token: null,\n    },\n  },\n})\n\n// 获取刚注册的 tokenizer。\nconst tokenizer = await getTokenizer(\"toy\")\n\nconsole.log(tokenizer.encode(\"a\"))\n```\n","readmeFilename":"README.md"}