{"bugs":{"url":"https://github.com/niieani/gpt-tokenizer/issues"},"dist":{"shasum":"19fa42314d15b69a1e82d3898336b5ba1f4f2c86","tarball":"https://registry.npmjs.org/gpt-tokenizer/-/gpt-tokenizer-3.0.1.tgz","fileCount":1122,"integrity":"sha512-5jdaspBq/w4sWw322SvQj1Fku+CN4OAfYZeeEg8U7CWtxBz+zkxZ3h0YOHD43ee+nZYZ5Ud70HRN0ANcdIj4qg==","signatures":[{"sig":"MEYCIQCnzyTRB68PuYcgvpdeTa2pDVPno00ajEwUwAUdfOO1xQIhAO15QXIJXIDo5xwdmUqzN+Tod1wYNTrmpNT99jdXDyXX","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":46090654},"main":"esm/main.js","name":"gpt-tokenizer","types":"./esm/main.d.ts","unpkg":"dist/cl100k_base.js","author":{"url":"https://github.com/niieani","name":"Bazyli Brzoska","email":"npm@invent.life"},"module":"esm/main.js","source":"src/main.ts","exports":{".":{"import":"./esm/main.js","require":"./cjs/main.js"},"./*":{"import":"./esm/*.js","require":"./cjs/*.js"},"./cjs":{"require":"./cjs/main.js"},"./cjs/*":{"require":"./cjs/*.js"},"./esm/*":{"import":"./esm/*.js"},"./data/*":{"import":"./data/*","require":"./data/*"},"./package.json":"./package.json"},"gitHead":"f77db29c1034fbf254b6162d20d4874fc4d158b9","license":"MIT","release":{"branches":["+([0-9])?(.{+([0-9]),x}).x","master",{"name":"main","channel":false},"next",{"name":"beta","prerelease":true},{"name":"alpha","prerelease":true}],"tagFormat":"${version}"},"scripts":{"test":"yarn test:format && yarn test:types && yarn test:lint && yarn test:code","build":"yarn build:cjs && yarn build:esm && yarn build:umd","clean":"git clean -dfX --exclude=node_modules src && beemo typescript:sync-project-refs","format":"yarn rrun prettier --write \"./{src,tests,.config}/**/!(*.d).{.js,jsx,ts,tsx,json,md}\"","codegen":"yarn codegen:bpe && yarn codegen:chat-enabled && yarn codegen:models","prepare":"rrun husky install .config/husky && beemo create-config && echo '\n**/*.gen.ts\nsrc/models.ts' >> .prettierignore","release":"beemo run-script release","build:cjs":"yarn rrun tsc --outDir cjs --module commonjs --target es2022 --project tsconfig-cjs.json","build:esm":"mkdir -p esm && echo '{\"name\": \"gpt-tokenizer\", \"type\": \"module\"}' > ./esm/package.json && yarn rrun tsc --outDir esm --target es2022","build:umd":"yarn build:umd:cl100k_base && yarn build:umd:p50k_base && yarn build:umd:p50k_edit && yarn build:umd:r50k_base && yarn build:umd:o200k_base","test:code":"vitest","test:lint":"rrun eslint 'src/*.{js,jsx,ts,tsx}'","test:types":"yarn rrun tsc --noEmit","codegen:bpe":"rm -rf src/bpeRanks && node --experimental-transform-types --import node-resolve-ts/register src/codegen/generateJsBpe.ts","test:format":"yarn rrun prettier --check \"./{src,tests,.config}/**/!(*.d).{.js,jsx,ts,tsx,json,md}\" --ignore-path .prettierignore","codegen:models":"rm -rf src/model && node --experimental-transform-types --import node-resolve-ts/register src/codegen/generateByModel.ts","postinstallDev":"yarn prepare","build:umd:p50k_base":"beemo webpack --entry='./src/encoding/p50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_p50k_base' --env 'filename=p50k_base.js'","build:umd:p50k_edit":"beemo webpack --entry='./src/encoding/p50k_edit.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_p50k_edit' --env 'filename=p50k_edit.js'","build:umd:r50k_base":"beemo webpack --entry='./src/encoding/r50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_r50k_base' --env 'filename=r50k_base.js'","build:umd:o200k_base":"beemo webpack --entry='./src/encoding/o200k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_o200k_base' --env 'filename=o200k_base.js'","codegen:chat-enabled":"rm -rf src/chat && node --experimental-transform-types --import node-resolve-ts/register src/codegen/generateChatEnabled.ts","build:umd:cl100k_base":"beemo webpack --entry='./src/main.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_cl100k_base' --env 'filename=cl100k_base.js'"},"_npmUser":{"name":"niieani","email":"npm@invent.life"},"homepage":"https://github.com/niieani/gpt-tokenizer#readme","keywords":["BPE","encoder","decoder","tokenizer","GPT","GPT-2","GPT-3","GPT-3.5","GPT-4","GPT-4o","NLP","Natural Language Processing","Text Generation","OpenAI","Machine Learning","ml"],"repository":{"url":"git+https://github.com/niieani/gpt-tokenizer.git","type":"git"},"_npmVersion":"8.19.4","description":"A pure JavaScript implementation of a BPE tokenizer (Encoder/Decoder) for GPT-2 / GPT-3 / GPT-4 and other OpenAI models","directories":{},"maintainers":[{"name":"niieani","email":"npm@invent.life"}],"resolutions":{"prettier":"^3","typescript":"5.8.3"},"_nodeVersion":"22.16.0","contributors":[{"url":"https://github.com/niieani","name":"Bazyli Brzoska","email":"npm@invent.life"}],"publishConfig":{"access":"public"},"_hasShrinkwrap":false,"packageManager":"yarn@4.9.2","devDependencies":{"vitest":"^3.2.2","devalue":"^5.1.1","@swc/cli":"^0.7.7","@swc/core":"^1.11.31","typescript":"^5.8.3","node-resolve-ts":"^1.0.2","@edge-runtime/vm":"^5.0.0","@niieani/scaffold":"^1.7.49"},"_npmOperationalInternal":{"tmp":"tmp/gpt-tokenizer_3.0.1_1749791051098_0.7489273387289539","host":"s3://npm-registry-packages-npm-production"},"_id":"gpt-tokenizer@3.0.1","version":"3.0.1"}