{"_id":"@agnai/web-tokenizers","_rev":"5-15b27cf4f356a5e5db81a06b39aa3541","name":"@agnai/web-tokenizers","dist-tags":{"latest":"0.1.6-pre3"},"versions":{"0.1.2":{"name":"@agnai/web-tokenizers","version":"0.1.2","keywords":["machine_learning","llm","nlp"],"license":"Apache-2.0","_id":"@agnai/web-tokenizers@0.1.2","maintainers":[{"name":"sceuick","email":"sceuick@gmail.com"}],"homepage":"https://github.com/mlc-ai/tokenizers-cpp/tree/main/web","bugs":{"url":"https://github.com/mlc-ai/tokenizers-cpp/issues"},"dist":{"shasum":"0c135aa6e1e586e782c8fed6981faccd6e88c070","tarball":"https://registry.npmjs.org/@agnai/web-tokenizers/-/web-tokenizers-0.1.2.tgz","fileCount":7,"integrity":"sha512-nmn9Oiv1gvUKGi+J53M2Zvm0iVnMrSveemWUD2H9uPY1fqd5K1EmcZPo7aXZ90Rrv7Z36DFyCrfmQF9huB6SkA==","signatures":[{"sig":"MEQCIDAF/MQDLTVZgDuMtSGI2Uu0ctEdnPrtyWZmTh7avWQIAiB1KPZl1z2LZDxfRTUBS6Kd9FyZMxO+flMFkdw5wQx5yg==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":4158298},"main":"lib/index.js","type":"module","types":"lib/index.d.ts","gitHead":"6e3a37e2ce4165fb70635a684ad300034fcb63dc","scripts":{"lint":"npx eslint .","build":"./build.sh; rollup -c"},"_npmUser":{"name":"sceuick","email":"sceuick@gmail.com"},"repository":{"url":"git+https://github.com/mlc-ai/tokenizers-cpp.git","type":"git"},"_npmVersion":"8.15.0","description":"| [NPM Package](https://www.npmjs.com/package/@mlc-ai/web-tokenizers) | [WebLLM](https://github.com/mlc-ai/web-llm) |","directories":{},"_nodeVersion":"16.17.0","publishConfig":{"access":"public"},"_hasShrinkwrap":false,"devDependencies":{"tslib":"^2.3.1","eslint":"^8.41.0","rollup":"^2.56.2","typescript":"^4.9.5","@rollup/plugin-wasm":"^5.1.2","@rollup/plugin-commonjs":"^20.0.0","@typescript-eslint/parser":"^5.59.6","rollup-plugin-typescript2":"^0.34.1","@rollup/plugin-node-resolve":"^13.0.4","@typescript-eslint/eslint-plugin":"^5.59.6"},"_npmOperationalInternal":{"tmp":"tmp/web-tokenizers_0.1.2_1691904583673_0.4845431704275971","host":"s3://npm-registry-packages"}},"0.1.3":{"name":"@agnai/web-tokenizers","version":"0.1.3","keywords":["machine_learning","llm","nlp"],"license":"Apache-2.0","_id":"@agnai/web-tokenizers@0.1.3","maintainers":[{"name":"sceuick","email":"sceuick@gmail.com"}],"homepage":"https://github.com/mlc-ai/tokenizers-cpp/tree/main/web","bugs":{"url":"https://github.com/mlc-ai/tokenizers-cpp/issues"},"dist":{"shasum":"b1b9b43d53a0bfeeee363d52d309a8e43fccacd6","tarball":"https://registry.npmjs.org/@agnai/web-tokenizers/-/web-tokenizers-0.1.3.tgz","fileCount":7,"integrity":"sha512-KlmTftToTtmb6aLVdne4NluS+POWputPF5J8v25UN/EQS+K9vahWEIe1NPRSFqBQclObkqHaj7JOnFrmnSm5MA==","signatures":[{"sig":"MEUCIElWx+LM22Yxk9rKA0WdWNI7XFZNuBvsaHv9VCMDJXGGAiEAgQn759lhR9O+ZA4KiI4X4CQQ1wSf1lGnfgfHj9UuUAQ=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":4158300},"main":"lib/index.js","type":"commonjs","types":"lib/index.d.ts","gitHead":"6e3a37e2ce4165fb70635a684ad300034fcb63dc","scripts":{"lint":"npx eslint .","build":"./build.sh; rollup -c"},"_npmUser":{"name":"sceuick","email":"sceuick@gmail.com"},"repository":{"url":"git+https://github.com/mlc-ai/tokenizers-cpp.git","type":"git"},"_npmVersion":"8.15.0","description":"| [NPM Package](https://www.npmjs.com/package/@mlc-ai/web-tokenizers) | [WebLLM](https://github.com/mlc-ai/web-llm) |","directories":{},"_nodeVersion":"16.17.0","publishConfig":{"access":"public"},"_hasShrinkwrap":false,"devDependencies":{"tslib":"^2.3.1","eslint":"^8.41.0","rollup":"^2.56.2","typescript":"^4.9.5","@rollup/plugin-wasm":"^5.1.2","@rollup/plugin-commonjs":"^20.0.0","@typescript-eslint/parser":"^5.59.6","rollup-plugin-typescript2":"^0.34.1","@rollup/plugin-node-resolve":"^13.0.4","@typescript-eslint/eslint-plugin":"^5.59.6"},"_npmOperationalInternal":{"tmp":"tmp/web-tokenizers_0.1.3_1691904868649_0.08884971140313547","host":"s3://npm-registry-packages"}},"0.1.6-pre":{"name":"@agnai/web-tokenizers","version":"0.1.6-pre","keywords":["machine_learning","llm","nlp"],"license":"Apache-2.0","_id":"@agnai/web-tokenizers@0.1.6-pre","maintainers":[{"name":"sceuick","email":"sceuick@gmail.com"}],"homepage":"https://github.com/mlc-ai/tokenizers-cpp/tree/main/web","bugs":{"url":"https://github.com/mlc-ai/tokenizers-cpp/issues"},"dist":{"shasum":"d96eaf8e7634fe51b12125c22c0796eca3634fe3","tarball":"https://registry.npmjs.org/@agnai/web-tokenizers/-/web-tokenizers-0.1.6-pre.tgz","fileCount":8,"integrity":"sha512-Bd3vBRxSp8NHo1PCVqRFdDetBR76tC3yk7nRyNLFtcjeRTMxKSr3Ka0dGJ6HFjNCvFNEZPisdipY5vaDyImtzw==","signatures":[{"sig":"MEQCIH7oTkrK8MWCoMYtMLFAXePszqjZUTIIoSE2ljgptAdYAiA6zFHBnHxww+FpvuLj/eoUS85nqfrlZ8DPv4F23vlXXw==","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":17041326},"main":"lib/index.js","type":"module","types":"lib/index.d.ts","gitHead":"55d53aa38dc8df7d9c8bd9ed50907e82ae83ce66","scripts":{"lint":"npx eslint .","build":"./build.sh; rollup -c"},"_npmUser":{"name":"sceuick","email":"sceuick@gmail.com"},"repository":{"url":"git+https://github.com/mlc-ai/tokenizers-cpp.git","type":"git"},"_npmVersion":"10.8.2","description":"| [NPM Package](https://www.npmjs.com/package/@mlc-ai/web-tokenizers) | [WebLLM](https://github.com/mlc-ai/web-llm) |","directories":{},"_nodeVersion":"20.18.1","_hasShrinkwrap":false,"devDependencies":{"tslib":"^2.3.1","eslint":"^8.41.0","rollup":"^2.56.2","typescript":"^4.9.5","@rollup/plugin-wasm":"^5.1.2","@rollup/plugin-commonjs":"^20.0.0","@typescript-eslint/parser":"^5.59.6","rollup-plugin-typescript2":"^0.34.1","@rollup/plugin-node-resolve":"^13.0.4","@typescript-eslint/eslint-plugin":"^5.59.6"},"_npmOperationalInternal":{"tmp":"tmp/web-tokenizers_0.1.6-pre_1765681518545_0.060549449463055405","host":"s3://npm-registry-packages-npm-production"}},"0.1.6-pre2":{"name":"@agnai/web-tokenizers","version":"0.1.6-pre2","keywords":["machine_learning","llm","nlp"],"license":"Apache-2.0","_id":"@agnai/web-tokenizers@0.1.6-pre2","maintainers":[{"name":"sceuick","email":"sceuick@gmail.com"}],"homepage":"https://github.com/mlc-ai/tokenizers-cpp/tree/main/web","bugs":{"url":"https://github.com/mlc-ai/tokenizers-cpp/issues"},"dist":{"shasum":"181225293a63bf6275b80c6bb111355bcf47754b","tarball":"https://registry.npmjs.org/@agnai/web-tokenizers/-/web-tokenizers-0.1.6-pre2.tgz","fileCount":9,"integrity":"sha512-gRi45yQKkWhZcAgXnrqSWfPnC88OlMy/PTfk217t5K2teNVNjXv1rEsXORgDK4JywrMud8lT3jn6iNgJxrjvBQ==","signatures":[{"sig":"MEUCIQC9Zl7FDQ5ygbnFvC9CaJEhGK+oiiNBfhiO4PWPJHuVUgIgOPICCeL7sO5tjkeKC63qi4P2LJDdfMtQZVtII3QOC9k=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":21280154},"main":"lib/index.js","type":"module","types":"lib/index.d.ts","exports":{".":{"common":"./lib/index.cjs","import":"./lib/index.js"}},"gitHead":"55d53aa38dc8df7d9c8bd9ed50907e82ae83ce66","scripts":{"lint":"npx eslint .","build":"./build.sh; rollup -c"},"_npmUser":{"name":"sceuick","email":"sceuick@gmail.com"},"repository":{"url":"git+https://github.com/mlc-ai/tokenizers-cpp.git","type":"git"},"_npmVersion":"10.8.2","description":"| [NPM Package](https://www.npmjs.com/package/@mlc-ai/web-tokenizers) | [WebLLM](https://github.com/mlc-ai/web-llm) |","directories":{},"_nodeVersion":"20.18.1","_hasShrinkwrap":false,"devDependencies":{"tslib":"^2.3.1","eslint":"^8.41.0","rollup":"^2.56.2","typescript":"^4.9.5","@rollup/plugin-wasm":"^5.1.2","@rollup/plugin-commonjs":"^20.0.0","@typescript-eslint/parser":"^5.59.6","rollup-plugin-typescript2":"^0.34.1","@rollup/plugin-node-resolve":"^13.0.4","@typescript-eslint/eslint-plugin":"^5.59.6"},"_npmOperationalInternal":{"tmp":"tmp/web-tokenizers_0.1.6-pre2_1765715520912_0.7650883845230878","host":"s3://npm-registry-packages-npm-production"}},"0.1.6-pre3":{"name":"@agnai/web-tokenizers","version":"0.1.6-pre3","description":"| [NPM Package](https://www.npmjs.com/package/@mlc-ai/web-tokenizers) | [WebLLM](https://github.com/mlc-ai/web-llm) |","main":"lib/index.js","types":"lib/index.d.ts","type":"module","scripts":{"build":"./build.sh; rollup -c","lint":"npx eslint ."},"exports":{".":{"require":"./lib/index.cjs","import":"./lib/index.js"}},"repository":{"type":"git","url":"git+https://github.com/mlc-ai/tokenizers-cpp.git"},"keywords":["machine_learning","llm","nlp"],"license":"Apache-2.0","homepage":"https://github.com/mlc-ai/tokenizers-cpp/tree/main/web","devDependencies":{"@rollup/plugin-commonjs":"^20.0.0","@rollup/plugin-node-resolve":"^13.0.4","@rollup/plugin-wasm":"^5.1.2","@typescript-eslint/eslint-plugin":"^5.59.6","@typescript-eslint/parser":"^5.59.6","eslint":"^8.41.0","rollup":"^2.56.2","rollup-plugin-typescript2":"^0.34.1","tslib":"^2.3.1","typescript":"^4.9.5"},"_id":"@agnai/web-tokenizers@0.1.6-pre3","gitHead":"55d53aa38dc8df7d9c8bd9ed50907e82ae83ce66","bugs":{"url":"https://github.com/mlc-ai/tokenizers-cpp/issues"},"_nodeVersion":"20.18.1","_npmVersion":"10.8.2","dist":{"integrity":"sha512-oJ2JqB+YB76ksZxFjtmZyWxrKxuKF50faSFIfrxMppijOH5EZwb7znwg/w4/EDFF0TE3s5jfOSUyqCSqAKQS+Q==","shasum":"f42eb9eaa95efe8ae49b9da39e0e956e3369b484","tarball":"https://registry.npmjs.org/@agnai/web-tokenizers/-/web-tokenizers-0.1.6-pre3.tgz","fileCount":9,"unpackedSize":21280155,"signatures":[{"keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U","sig":"MEUCIQD9UdgIvVR9pfMSUSnIs6n71PZMTjdMlbE5mhI26U5ZegIgCCIxuJrq/MtUOwTANDWiLXsTRSYfxMnU1wdoogbQAmQ="}]},"_npmUser":{"name":"sceuick","email":"sceuick@gmail.com"},"directories":{},"maintainers":[{"name":"sceuick","email":"sceuick@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages-npm-production","tmp":"tmp/web-tokenizers_0.1.6-pre3_1765716255459_0.817846659388404"},"_hasShrinkwrap":false}},"time":{"created":"2023-08-13T05:29:43.589Z","modified":"2025-12-14T12:44:15.977Z","0.1.2":"2023-08-13T05:29:44.021Z","0.1.3":"2023-08-13T05:34:28.994Z","0.1.6-pre":"2025-12-14T03:05:18.837Z","0.1.6-pre2":"2025-12-14T12:32:01.191Z","0.1.6-pre3":"2025-12-14T12:44:15.743Z"},"bugs":{"url":"https://github.com/mlc-ai/tokenizers-cpp/issues"},"license":"Apache-2.0","homepage":"https://github.com/mlc-ai/tokenizers-cpp/tree/main/web","keywords":["machine_learning","llm","nlp"],"repository":{"type":"git","url":"git+https://github.com/mlc-ai/tokenizers-cpp.git"},"description":"| [NPM Package](https://www.npmjs.com/package/@mlc-ai/web-tokenizers) | [WebLLM](https://github.com/mlc-ai/web-llm) |","maintainers":[{"name":"sceuick","email":"sceuick@gmail.com"}],"readme":"# WebTokenizers\n\n| [NPM Package](https://www.npmjs.com/package/@mlc-ai/web-tokenizers) | [WebLLM](https://github.com/mlc-ai/web-llm) |\n\nWebTokenizers is a javascript binding library that can be universally deployed.\nIt wraps and binds the [HuggingFace tokenizers library](https://github.com/huggingface/tokenizers)\nand [sentencepiece](https://github.com/google/sentencepiece) and provides a minimum common interface\nin havascript.\n\n## Build from Srouce\n\n```bash\nsource /path/to/emsdk_env.sh\nnpm install\nnpm run build\n```\n\nTo try out the test webpage\n```bash\ncd tests\nnpm start\n```\n","readmeFilename":"README.md"}