{"_id":"@ai-sdk-tool/eval","_rev":"12-59fe9504bb7141067977099c8f74bbd4","name":"@ai-sdk-tool/eval","dist-tags":{"canary":"1.0.0-canary.1","latest":"1.1.0"},"versions":{"0.1.1":{"name":"@ai-sdk-tool/eval","version":"0.1.1","_id":"@ai-sdk-tool/eval@0.1.1","maintainers":[{"name":"minpeter","email":"kali2005611@gmail.com"}],"dist":{"shasum":"baabd4f352f6c273756f90f6baecac473d5bf38b","tarball":"https://registry.npmjs.org/@ai-sdk-tool/eval/-/eval-0.1.1.tgz","fileCount":18,"integrity":"sha512-mcBUa+5sc3Y8qdd0DdGAJzOfS/yebKMAILl3s2REkqa5tuwPz1EYzSqxasITKjM36vN5rbFN+LO7K9lYOEauow==","signatures":[{"sig":"MEUCIGY4L201ncW4bC2LEkXmear/Mr4otzQKdtwxTT/Msx5TAiEAzPxu7d1fijr/1l/bMLZnLF5q8Eu3AWMr2ElfRq8pxbQ=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":1535712},"main":"./dist/index.js","type":"module","_from":"file:ai-sdk-tool-eval-0.1.1.tgz","types":"./dist/index.d.ts","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","default":"./dist/index.js","require":"./dist/index.cjs"}},"scripts":{"dev":"tsup --watch","test":"pnpm run test:e2e","build":"tsup --clean","test:e2e":"tsup src/run-test.ts --format esm --clean --out-dir dist --no-config && node dist/run-test.js","check-types":"tsc --noEmit"},"_npmUser":{"name":"minpeter","email":"kali2005611@gmail.com"},"_resolved":"/tmp/5df38e55d1e9988eab76d76aa1051d89/ai-sdk-tool-eval-0.1.1.tgz","_integrity":"sha512-mcBUa+5sc3Y8qdd0DdGAJzOfS/yebKMAILl3s2REkqa5tuwPz1EYzSqxasITKjM36vN5rbFN+LO7K9lYOEauow==","_npmVersion":"10.8.2","description":"This package provides a standardized, extensible, and reproducible way to benchmark and evaluate the performance of Language Models (`LanguageModel` instances) within the Vercel AI SDK ecosystem.","directories":{},"_nodeVersion":"18.20.8","dependencies":{"ajv":"^8.17.1","zod":"^4.1.1"},"publishConfig":{"access":"public"},"_hasShrinkwrap":false,"devDependencies":{"tsup":"^8.5.0","typescript":"^5.9.2","@types/node":"^24.3.0"},"peerDependencies":{"ai":">=3.0.0"},"_npmOperationalInternal":{"tmp":"tmp/eval_0.1.1_1756081382062_0.05247350840398535","host":"s3://npm-registry-packages-npm-production"}},"0.1.2":{"name":"@ai-sdk-tool/eval","version":"0.1.2","_id":"@ai-sdk-tool/eval@0.1.2","maintainers":[{"name":"minpeter","email":"kali2005611@gmail.com"}],"dist":{"shasum":"9a9286068ef66f3c560d229d637d67c2443202b0","tarball":"https://registry.npmjs.org/@ai-sdk-tool/eval/-/eval-0.1.2.tgz","fileCount":18,"integrity":"sha512-soCi3RdNXdAV0Qz8int0HTQMjRBJl/o8ScfwWJfjbDMGotBsbRKN6ukoghSvjqHBzTSnin5mabvuWkyBRPvZ+Q==","signatures":[{"sig":"MEUCIQCbOXPQ3abiYFuU6lYkvqwaBLsM+2N2gHmRMJsRgOSxAQIgE+9piN26J9GOTohv0I3bwK8om/d0R6ZnCXKKIf1NTbM=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":1537298},"main":"./dist/index.js","type":"module","_from":"file:ai-sdk-tool-eval-0.1.2.tgz","types":"./dist/index.d.ts","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","default":"./dist/index.js","require":"./dist/index.cjs"}},"scripts":{"dev":"tsup --watch","test":"pnpm run test:e2e","build":"tsup --clean","test:e2e":"tsup src/run-test.ts --format esm --clean --out-dir dist --no-config && node dist/run-test.js","check-types":"tsc --noEmit"},"_npmUser":{"name":"minpeter","email":"kali2005611@gmail.com"},"_resolved":"/tmp/ca3904a12a0e9472a1a4574682c677ab/ai-sdk-tool-eval-0.1.2.tgz","_integrity":"sha512-soCi3RdNXdAV0Qz8int0HTQMjRBJl/o8ScfwWJfjbDMGotBsbRKN6ukoghSvjqHBzTSnin5mabvuWkyBRPvZ+Q==","_npmVersion":"10.8.2","description":"This package provides a standardized, extensible, and reproducible way to benchmark and evaluate the performance of Language Models (`LanguageModel` instances) within the Vercel AI SDK ecosystem.","directories":{},"_nodeVersion":"18.20.8","dependencies":{"ajv":"^8.17.1","zod":"^4.1.3"},"publishConfig":{"access":"public"},"_hasShrinkwrap":false,"devDependencies":{"tsup":"^8.5.0","typescript":"^5.9.2","@types/node":"^24.3.0"},"peerDependencies":{"ai":">=3.0.0"},"_npmOperationalInternal":{"tmp":"tmp/eval_0.1.2_1756303190198_0.32578251427670746","host":"s3://npm-registry-packages-npm-production"}},"0.1.3":{"name":"@ai-sdk-tool/eval","version":"0.1.3","_id":"@ai-sdk-tool/eval@0.1.3","maintainers":[{"name":"minpeter","email":"kali2005611@gmail.com"}],"dist":{"shasum":"a69a324fe9870c5b5066830955b3b866671518d1","tarball":"https://registry.npmjs.org/@ai-sdk-tool/eval/-/eval-0.1.3.tgz","fileCount":18,"integrity":"sha512-8/Q6XObob9TdWLooEGKt7A62B8qstF92pP1oTtyge/14UD7supzbsF6tBZQIhDCytF7lKUti7szIWIsh4fJJKQ==","signatures":[{"sig":"MEQCIDZDjfrezRowGZfZpGolMlBfWeqggyXKibZ0ALN5F7+IAiAxzDJtXfYLJD9uIyhGe//3GHQRDDvCIhvwiGvqrBRTLw==","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":1652780},"main":"./dist/index.js","type":"module","_from":"file:ai-sdk-tool-eval-0.1.3.tgz","types":"./dist/index.d.ts","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","default":"./dist/index.js","require":"./dist/index.cjs"}},"scripts":{"dev":"tsup --watch","test":"pnpm run test:e2e","build":"tsup --clean","test:e2e":"tsup src/run-test.ts --format esm --clean --out-dir dist --no-config && node dist/run-test.js","check-types":"tsc --noEmit"},"_npmUser":{"name":"minpeter","email":"kali2005611@gmail.com"},"_resolved":"/tmp/149d472db72baffd8b16e59d155bad52/ai-sdk-tool-eval-0.1.3.tgz","_integrity":"sha512-8/Q6XObob9TdWLooEGKt7A62B8qstF92pP1oTtyge/14UD7supzbsF6tBZQIhDCytF7lKUti7szIWIsh4fJJKQ==","_npmVersion":"10.8.2","description":"This package provides a standardized, extensible, and reproducible way to benchmark and evaluate the performance of Language Models (`LanguageModel` instances) within the Vercel AI SDK ecosystem.","directories":{},"_nodeVersion":"18.20.8","dependencies":{"ajv":"^8.17.1","zod":"^4.1.5"},"publishConfig":{"access":"public"},"_hasShrinkwrap":false,"devDependencies":{"tsup":"^8.5.0","typescript":"^5.9.2","@types/node":"^24.3.0"},"peerDependencies":{"ai":">=3.0.0"},"_npmOperationalInternal":{"tmp":"tmp/eval_0.1.3_1756438235120_0.13150135953501896","host":"s3://npm-registry-packages-npm-production"}},"0.1.4":{"name":"@ai-sdk-tool/eval","version":"0.1.4","_id":"@ai-sdk-tool/eval@0.1.4","maintainers":[{"name":"minpeter","email":"kali2005611@gmail.com"}],"dist":{"shasum":"1407033e589eefaf61a7717e87d39abd65c62a94","tarball":"https://registry.npmjs.org/@ai-sdk-tool/eval/-/eval-0.1.4.tgz","fileCount":18,"integrity":"sha512-e1McQqW9piHgnaDOQ761Z/3N3rqBgAIisIjLJ6iSs+MatpHrEYDKFIErDX1imIUeJSMmWJOWtdw/zd8C16sgIg==","signatures":[{"sig":"MEQCIAIj73Bl+149spAmN2lue5KqJ6AgCstRZ8cYb0n9pAUHAiB2APxsr6vlT7i1iYbkbnYL7l7oNqQAPRmuJsVEZ4wZ9A==","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":1667949},"main":"./dist/index.js","type":"module","_from":"file:ai-sdk-tool-eval-0.1.4.tgz","types":"./dist/index.d.ts","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","default":"./dist/index.js","require":"./dist/index.cjs"}},"scripts":{"dev":"tsup --watch","lint":"eslint src --ext .ts,.tsx","test":"pnpm run test:e2e","build":"tsup --clean","lint:fix":"eslint src --ext .ts,.tsx --fix","test:e2e":"tsup src/run-test.ts --format esm --clean --out-dir dist --no-config && node dist/run-test.js","check-types":"tsc --noEmit"},"_npmUser":{"name":"minpeter","email":"kali2005611@gmail.com"},"_resolved":"/tmp/0f26fde88ff1bfbb426e66979054483d/ai-sdk-tool-eval-0.1.4.tgz","_integrity":"sha512-e1McQqW9piHgnaDOQ761Z/3N3rqBgAIisIjLJ6iSs+MatpHrEYDKFIErDX1imIUeJSMmWJOWtdw/zd8C16sgIg==","_npmVersion":"10.8.2","description":"[![npm](https://img.shields.io/npm/v/@ai-sdk-tool/eval)](https://www.npmjs.com/package/@ai-sdk-tool/eval) [![npm](https://img.shields.io/npm/dt/@ai-sdk-tool/eval)](https://www.npmjs.com/package/@ai-sdk-tool/eval)","directories":{},"_nodeVersion":"18.20.8","dependencies":{"ajv":"^8.17.1","zod":"^4.1.5","@ai-sdk/provider":"2.0.0"},"publishConfig":{"access":"public"},"_hasShrinkwrap":false,"devDependencies":{"tsup":"^8.5.0","typescript":"^5.9.2","@types/node":"^24.3.0"},"peerDependencies":{"ai":">=3.0.0"},"_npmOperationalInternal":{"tmp":"tmp/eval_0.1.4_1756563600764_0.4745549313542943","host":"s3://npm-registry-packages-npm-production"}},"0.1.5":{"name":"@ai-sdk-tool/eval","version":"0.1.5","_id":"@ai-sdk-tool/eval@0.1.5","maintainers":[{"name":"minpeter","email":"kali2005611@gmail.com"}],"dist":{"shasum":"6e97a23f900852630bf407426f627df4642d67dc","tarball":"https://registry.npmjs.org/@ai-sdk-tool/eval/-/eval-0.1.5.tgz","fileCount":18,"integrity":"sha512-A6npEgen3rAuxjJVsYQmkz5PMirAQFCD6ssoo0hLOnql/byOrp5VBr7kYFJ/Zy8D08JyPnn9YwELRaXfxmmcMg==","signatures":[{"sig":"MEUCIBpPMipbKOhOroABZK6mZKtjw6hR+Xj/3En4hUo3XZhsAiEAxKTJ93HORv3t4qB4II4rxL0d8lYfGkJ+kmfbmS36bi8=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":1672221},"main":"./dist/index.js","type":"module","_from":"file:ai-sdk-tool-eval-0.1.5.tgz","types":"./dist/index.d.ts","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","default":"./dist/index.js","require":"./dist/index.cjs"}},"scripts":{"dev":"tsup --watch","lint":"eslint src --ext .ts,.tsx","test":"pnpm run test:e2e","build":"tsup --clean","lint:fix":"eslint src --ext .ts,.tsx --fix","test:e2e":"tsup src/run-test.ts --format esm --clean --out-dir dist --no-config && node dist/run-test.js","check-types":"tsc --noEmit"},"_npmUser":{"name":"minpeter","email":"kali2005611@gmail.com"},"_resolved":"/tmp/3e70ebb75f81a0e619d991b62430c7eb/ai-sdk-tool-eval-0.1.5.tgz","_integrity":"sha512-A6npEgen3rAuxjJVsYQmkz5PMirAQFCD6ssoo0hLOnql/byOrp5VBr7kYFJ/Zy8D08JyPnn9YwELRaXfxmmcMg==","_npmVersion":"10.8.2","description":"[![npm](https://img.shields.io/npm/v/@ai-sdk-tool/eval)](https://www.npmjs.com/package/@ai-sdk-tool/eval) [![npm](https://img.shields.io/npm/dt/@ai-sdk-tool/eval)](https://www.npmjs.com/package/@ai-sdk-tool/eval)","directories":{},"_nodeVersion":"18.20.8","dependencies":{"ajv":"^8.17.1","zod":"^4.1.5","@ai-sdk/provider":"2.0.0"},"publishConfig":{"access":"public"},"_hasShrinkwrap":false,"devDependencies":{"tsup":"^8.5.0","typescript":"^5.9.2","@types/node":"^24.3.0"},"peerDependencies":{"ai":">=3.0.0"},"_npmOperationalInternal":{"tmp":"tmp/eval_0.1.5_1756627634283_0.5890762370275697","host":"s3://npm-registry-packages-npm-production"}},"0.1.6":{"name":"@ai-sdk-tool/eval","version":"0.1.6","license":"Apache-2.0","_id":"@ai-sdk-tool/eval@0.1.6","maintainers":[{"name":"minpeter","email":"kali2005611@gmail.com"}],"dist":{"shasum":"b5737af838270a50f11d1258f5effc273f64fb49","tarball":"https://registry.npmjs.org/@ai-sdk-tool/eval/-/eval-0.1.6.tgz","fileCount":19,"integrity":"sha512-AKUFWj2WYLqkqyXZe067y6DEuvJmeRNOo5uyGCc9HDXw8BhbASYFcOcmUnnI3AQ5MHH39lLwJZm8gbI8R++XDQ==","signatures":[{"sig":"MEUCIFlsKMYhYCsEEMvUI4coPYgYm3SHiJYdCs+lWL6QllvvAiEAqXlQiHCB/Jbj3ezMSFkn+n/Nhze/7IqwkjPnSeN9PJk=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":1672952},"main":"./dist/index.js","type":"module","_from":"file:ai-sdk-tool-eval-0.1.6.tgz","types":"./dist/index.d.ts","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","default":"./dist/index.js","require":"./dist/index.cjs"}},"scripts":{"dev":"tsup --watch","lint":"eslint src --ext .ts,.tsx","test":"pnpm run test:e2e","build":"tsup --clean","lint:fix":"eslint src --ext .ts,.tsx --fix","test:e2e":"tsup src/run-test.ts --format esm --clean --out-dir dist --no-config && node dist/run-test.js","check-types":"tsc --noEmit"},"_npmUser":{"name":"minpeter","email":"kali2005611@gmail.com"},"_resolved":"/tmp/d357b07fd0d612a68d697a369d4a02d8/ai-sdk-tool-eval-0.1.6.tgz","_integrity":"sha512-AKUFWj2WYLqkqyXZe067y6DEuvJmeRNOo5uyGCc9HDXw8BhbASYFcOcmUnnI3AQ5MHH39lLwJZm8gbI8R++XDQ==","_npmVersion":"10.8.2","description":"[![npm](https://img.shields.io/npm/v/@ai-sdk-tool/eval)](https://www.npmjs.com/package/@ai-sdk-tool/eval) [![npm](https://img.shields.io/npm/dt/@ai-sdk-tool/eval)](https://www.npmjs.com/package/@ai-sdk-tool/eval)","directories":{},"_nodeVersion":"18.20.8","dependencies":{"ajv":"^8.17.1","zod":"^4.1.5","@ai-sdk/provider":"2.0.0"},"publishConfig":{"access":"public"},"_hasShrinkwrap":false,"devDependencies":{"tsup":"^8.5.0","typescript":"^5.9.2","@types/node":"^24.3.0"},"peerDependencies":{"ai":">=3.0.0"},"_npmOperationalInternal":{"tmp":"tmp/eval_0.1.6_1756696760108_0.35800985967663435","host":"s3://npm-registry-packages-npm-production"}},"0.1.7":{"name":"@ai-sdk-tool/eval","version":"0.1.7","license":"Apache-2.0","_id":"@ai-sdk-tool/eval@0.1.7","maintainers":[{"name":"minpeter","email":"kali2005611@gmail.com"}],"dist":{"shasum":"e4cb6e7f51692feef1efaf7aa6af453dd987a09c","tarball":"https://registry.npmjs.org/@ai-sdk-tool/eval/-/eval-0.1.7.tgz","fileCount":19,"integrity":"sha512-fkxjesNiidf7DBoHFLExZ3t+WwPir3ifF9XZnMnxP5iI+nTo8LPaQMM8Buqp/L0vyc5jXJmTaJgHD2hOOjAdow==","signatures":[{"sig":"MEUCIHUOKq0b6RwwG/HjOOGtU8YA3fVCK0rKFTEhhWoYdrbVAiEAuMIsH7UudYGlft1JIMsvfUvr1Uudl2k2d1OgWZfbs6E=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":1673398},"main":"./dist/index.js","type":"module","_from":"file:ai-sdk-tool-eval-0.1.7.tgz","types":"./dist/index.d.ts","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","default":"./dist/index.js","require":"./dist/index.cjs"}},"scripts":{"dev":"tsup --watch","lint":"eslint src --ext .ts,.tsx","test":"pnpm run test:e2e","build":"tsup --clean","lint:fix":"eslint src --ext .ts,.tsx --fix","test:e2e":"tsup src/run-test.ts --format esm --clean --out-dir dist --no-config && node dist/run-test.js","check-types":"tsc --noEmit"},"_npmUser":{"name":"minpeter","email":"kali2005611@gmail.com"},"_resolved":"/tmp/8d5ddba19adc8278b4c02afd47bd4e94/ai-sdk-tool-eval-0.1.7.tgz","_integrity":"sha512-fkxjesNiidf7DBoHFLExZ3t+WwPir3ifF9XZnMnxP5iI+nTo8LPaQMM8Buqp/L0vyc5jXJmTaJgHD2hOOjAdow==","_npmVersion":"10.8.2","description":"[![npm](https://img.shields.io/npm/v/@ai-sdk-tool/eval)](https://www.npmjs.com/package/@ai-sdk-tool/eval) [![npm](https://img.shields.io/npm/dt/@ai-sdk-tool/eval)](https://www.npmjs.com/package/@ai-sdk-tool/eval)","directories":{},"_nodeVersion":"18.20.8","dependencies":{"ajv":"^8.17.1","zod":"^4.1.5","@ai-sdk/provider":"2.0.0"},"publishConfig":{"access":"public"},"_hasShrinkwrap":false,"devDependencies":{"tsup":"^8.5.0","typescript":"^5.9.2","@types/node":"^24.3.0"},"peerDependencies":{"ai":">=3.0.0"},"_npmOperationalInternal":{"tmp":"tmp/eval_0.1.7_1756824998656_0.7591718871071085","host":"s3://npm-registry-packages-npm-production"}},"0.1.8":{"name":"@ai-sdk-tool/eval","version":"0.1.8","license":"Apache-2.0","_id":"@ai-sdk-tool/eval@0.1.8","maintainers":[{"name":"minpeter","email":"kali2005611@gmail.com"}],"dist":{"shasum":"f4cdbe6c825fce2bd461d7ed299689b7076506f4","tarball":"https://registry.npmjs.org/@ai-sdk-tool/eval/-/eval-0.1.8.tgz","fileCount":19,"integrity":"sha512-RU4WH0OVVqle2EmqtLwmV3O1hRdrVCIrielzyVPQ+fqyvFyyLz/7C3uUyoJWDOzeSb1JgLPsW69Xdzi2BNU6gA==","signatures":[{"sig":"MEYCIQDVKWw3QuKTwxMPFGGSM+OIqWIiONiHN5c0SIBNjsCX9wIhAK29HOLMDTtvrofO7XHw83bRf9Q3AFisuw9spMWB8i0Z","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":1710020},"main":"./dist/index.js","type":"module","_from":"file:ai-sdk-tool-eval-0.1.8.tgz","types":"./dist/index.d.ts","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","default":"./dist/index.js","require":"./dist/index.cjs"}},"scripts":{"dev":"tsup --watch","lint":"eslint src --ext .ts,.tsx","test":"pnpm run test:e2e","build":"tsup --clean","lint:fix":"eslint src --ext .ts,.tsx --fix","test:e2e":"tsup src/run-test.ts --format esm --clean --out-dir dist --no-config && node dist/run-test.js","check-types":"tsc --noEmit"},"_npmUser":{"name":"minpeter","email":"kali2005611@gmail.com"},"_resolved":"/tmp/bf85528c7f1cad69102cb9ca8929a2e6/ai-sdk-tool-eval-0.1.8.tgz","_integrity":"sha512-RU4WH0OVVqle2EmqtLwmV3O1hRdrVCIrielzyVPQ+fqyvFyyLz/7C3uUyoJWDOzeSb1JgLPsW69Xdzi2BNU6gA==","_npmVersion":"10.8.2","description":"[![npm](https://img.shields.io/npm/v/@ai-sdk-tool/eval)](https://www.npmjs.com/package/@ai-sdk-tool/eval) [![npm](https://img.shields.io/npm/dt/@ai-sdk-tool/eval)](https://www.npmjs.com/package/@ai-sdk-tool/eval)","directories":{},"_nodeVersion":"18.20.8","dependencies":{"ajv":"^8.17.1","zod":"^4.1.5","@ai-sdk/provider":"2.0.0"},"publishConfig":{"access":"public"},"_hasShrinkwrap":false,"devDependencies":{"tsup":"^8.5.0","typescript":"^5.9.2","@types/node":"^24.3.0"},"peerDependencies":{"ai":">=3.0.0"},"_npmOperationalInternal":{"tmp":"tmp/eval_0.1.8_1756959423425_0.6127917190136798","host":"s3://npm-registry-packages-npm-production"}},"1.0.0-canary.0":{"name":"@ai-sdk-tool/eval","version":"1.0.0-canary.0","license":"Apache-2.0","_id":"@ai-sdk-tool/eval@1.0.0-canary.0","maintainers":[{"name":"minpeter","email":"kali2005611@gmail.com"}],"dist":{"shasum":"658ff9cb774356b8eae09f790c685398e1dbfb59","tarball":"https://registry.npmjs.org/@ai-sdk-tool/eval/-/eval-1.0.0-canary.0.tgz","fileCount":19,"integrity":"sha512-5lg13+9PpPPhWTa7Kxd8eQjyjjpaD029V5Q9wCZ1tGK99yPtTnXKeUxqPYXemTp4V5AJZ/9SslirFFbntCgfRQ==","signatures":[{"sig":"MEYCIQCla1hWRnYzfmSXc13Anqm5bctaUW3/ar1x6gsjJL3P8QIhAINn72Gp58PzoYx6NZ/QLbwjKjTs7r4c7QckZSGOIsea","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":1768313},"main":"./dist/index.js","type":"module","_from":"file:ai-sdk-tool-eval-1.0.0-canary.0.tgz","types":"./dist/index.d.ts","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","default":"./dist/index.js","require":"./dist/index.cjs"}},"scripts":{"dev":"tsup --watch","build":"tsup --clean","test:e2e":"tsup src/run-test.ts --format esm --clean --out-dir dist --no-config && node dist/run-test.js","typecheck":"tsc --noEmit"},"_npmUser":{"name":"minpeter","email":"kali2005611@gmail.com"},"_resolved":"/tmp/6255ff2f4774aee777d99488945515d7/ai-sdk-tool-eval-1.0.0-canary.0.tgz","_integrity":"sha512-5lg13+9PpPPhWTa7Kxd8eQjyjjpaD029V5Q9wCZ1tGK99yPtTnXKeUxqPYXemTp4V5AJZ/9SslirFFbntCgfRQ==","_npmVersion":"10.9.4","description":"[![npm](https://img.shields.io/npm/v/@ai-sdk-tool/eval)](https://www.npmjs.com/package/@ai-sdk-tool/eval) [![npm](https://img.shields.io/npm/dt/@ai-sdk-tool/eval)](https://www.npmjs.com/package/@ai-sdk-tool/eval)","directories":{},"_nodeVersion":"22.21.1","dependencies":{"ajv":"^8.17.1","zod":"^4.1.12","@ai-sdk/provider":"3.0.0-beta.14"},"publishConfig":{"access":"public"},"_hasShrinkwrap":false,"readmeFilename":"README.md","devDependencies":{"tsup":"^8.5.0","typescript":"^5.9.3","@types/node":"^24.10.0"},"peerDependencies":{"ai":">=3.0.0"},"_npmOperationalInternal":{"tmp":"tmp/eval_1.0.0-canary.0_1762169063749_0.5301916021078885","host":"s3://npm-registry-packages-npm-production"}},"1.0.0-canary.1":{"name":"@ai-sdk-tool/eval","version":"1.0.0-canary.1","license":"Apache-2.0","_id":"@ai-sdk-tool/eval@1.0.0-canary.1","maintainers":[{"name":"minpeter","email":"kali2005611@gmail.com"}],"homepage":"https://github.com/minpeter/ai-sdk-tool-call-middleware#readme","bugs":{"url":"https://github.com/minpeter/ai-sdk-tool-call-middleware/issues"},"dist":{"shasum":"eb0b052e61ee4cdce4648ba9bc27e2cd270921e7","tarball":"https://registry.npmjs.org/@ai-sdk-tool/eval/-/eval-1.0.0-canary.1.tgz","fileCount":21,"integrity":"sha512-rJ878p3Zmar/4EQLYMT11wvpuP6PCqJ0cAwpcNFJt8NU/7e92DT5vSiFQXp/XAiOum2SWpv+bVACY0plBJlHOQ==","signatures":[{"sig":"MEUCIQDAcUmMYjT1z0y5CAPVU9GifFf2w2gxJwT2Y7AMjC3L7gIgZAljQIhIuiCTXipaV4UGPlNp2juwxgHCnXtsJZHoerY=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"attestations":{"url":"https://registry.npmjs.org/-/npm/v1/attestations/@ai-sdk-tool%2feval@1.0.0-canary.1","provenance":{"predicateType":"https://slsa.dev/provenance/v1"}},"unpackedSize":7482841},"main":"./dist/index.js","type":"module","_from":"file:ai-sdk-tool-eval-1.0.0-canary.1.tgz","types":"./dist/index.d.ts","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","default":"./dist/index.js","require":"./dist/index.cjs"}},"scripts":{"build":"pnpm clean && tsup --tsconfig tsconfig.build.json","clean":"rm -rf dist *.tsbuildinfo","test:e2e":"tsup src/run-test.ts --format esm --clean --out-dir dist --no-config && node dist/run-test.js","typecheck":"tsc --noEmit","build:watch":"pnpm clean && tsup --watch --tsconfig tsconfig.build.json"},"_npmUser":{"name":"GitHub Actions","email":"npm-oidc-no-reply@github.com","trustedPublisher":{"id":"github","oidcConfigId":"oidc:1d4deafb-0afd-43dc-8fce-d31ba032c66f"}},"_resolved":"/tmp/9c29cd2bae41fc576b44f7a0c3ffcbca/ai-sdk-tool-eval-1.0.0-canary.1.tgz","_integrity":"sha512-rJ878p3Zmar/4EQLYMT11wvpuP6PCqJ0cAwpcNFJt8NU/7e92DT5vSiFQXp/XAiOum2SWpv+bVACY0plBJlHOQ==","repository":{"url":"git+https://github.com/minpeter/ai-sdk-tool-call-middleware.git","type":"git","directory":"packages/eval"},"_npmVersion":"11.6.2","description":"Benchmarking and evaluation tools for AI SDK","directories":{},"_nodeVersion":"24.12.0","dependencies":{"ai":"6.0.3","ajv":"^8.17.1","zod":"^4.2.1","@ai-sdk/provider":"3.0.0"},"publishConfig":{"access":"public"},"_hasShrinkwrap":false,"readmeFilename":"README.md","devDependencies":{"tsup":"^8.5.1","typescript":"^5.9.3","@types/node":"^25.0.3","@ai-sdkx/tsconfig":"0.0.1-canary.0"},"_npmOperationalInternal":{"tmp":"tmp/eval_1.0.0-canary.1_1766742372002_0.001678049641001822","host":"s3://npm-registry-packages-npm-production"}},"1.0.0":{"name":"@ai-sdk-tool/eval","version":"1.0.0","license":"Apache-2.0","_id":"@ai-sdk-tool/eval@1.0.0","maintainers":[{"name":"minpeter","email":"kali2005611@gmail.com"}],"homepage":"https://github.com/minpeter/ai-sdk-tool-call-middleware#readme","bugs":{"url":"https://github.com/minpeter/ai-sdk-tool-call-middleware/issues"},"dist":{"shasum":"53ae798653ab02dc3eb8fb8e328af802a6e8b306","tarball":"https://registry.npmjs.org/@ai-sdk-tool/eval/-/eval-1.0.0.tgz","fileCount":21,"integrity":"sha512-JMFl8zaPVqoQkhiX/2DWpkzWQ0CUrQtd1Vpg9ma+R0/yPKm9nn0fGv7D1rSAs87hMP81u1DlIZQl5tzgD1wlfw==","signatures":[{"sig":"MEQCICqpyi8wjA1T5x0+F//fmjYBE+YJawRV0rUAJt4IFZMFAiAWUqJzqZPtXhwpaNJ4Z4IJXSVzLP67wAf8QJ/4IQw7bg==","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"attestations":{"url":"https://registry.npmjs.org/-/npm/v1/attestations/@ai-sdk-tool%2feval@1.0.0","provenance":{"predicateType":"https://slsa.dev/provenance/v1"}},"unpackedSize":7592726},"main":"./dist/index.js","type":"module","_from":"file:ai-sdk-tool-eval-1.0.0.tgz","types":"./dist/index.d.ts","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","default":"./dist/index.js","require":"./dist/index.cjs"}},"scripts":{"build":"pnpm clean && tsup --tsconfig tsconfig.build.json","clean":"rm -rf dist *.tsbuildinfo","test:e2e":"tsup src/run-test.ts --format esm --clean --out-dir dist --no-config && node dist/run-test.js","typecheck":"tsc --noEmit","build:watch":"pnpm clean && tsup --watch --tsconfig tsconfig.build.json"},"_npmUser":{"name":"GitHub Actions","email":"npm-oidc-no-reply@github.com","trustedPublisher":{"id":"github","oidcConfigId":"oidc:1d4deafb-0afd-43dc-8fce-d31ba032c66f"}},"_resolved":"/tmp/badbf411253464ce478999d694719cbc/ai-sdk-tool-eval-1.0.0.tgz","_integrity":"sha512-JMFl8zaPVqoQkhiX/2DWpkzWQ0CUrQtd1Vpg9ma+R0/yPKm9nn0fGv7D1rSAs87hMP81u1DlIZQl5tzgD1wlfw==","repository":{"url":"git+https://github.com/minpeter/ai-sdk-tool-call-middleware.git","type":"git","directory":"packages/eval"},"_npmVersion":"11.6.2","description":"Benchmarking and evaluation tools for AI SDK","directories":{},"_nodeVersion":"24.12.0","dependencies":{"ai":"6.0.5","ajv":"^8.17.1","zod":"^4.3.4","@ai-sdk/provider":"3.0.1","@ai-sdk-tool/middleware":"0.0.1"},"publishConfig":{"access":"public"},"_hasShrinkwrap":false,"devDependencies":{"tsup":"^8.5.1","typescript":"^5.9.3","@types/node":"^25.0.3","@ai-sdkx/tsconfig":"0.0.1"},"_npmOperationalInternal":{"tmp":"tmp/eval_1.0.0_1767364053202_0.6421083475365592","host":"s3://npm-registry-packages-npm-production"}},"1.1.0":{"name":"@ai-sdk-tool/eval","version":"1.1.0","description":"Benchmarking and evaluation tools for AI SDK","license":"Apache-2.0","type":"module","repository":{"type":"git","url":"git+https://github.com/minpeter/ai-sdk-tool-call-middleware.git","directory":"packages/eval"},"main":"./dist/index.js","types":"./dist/index.d.ts","publishConfig":{"access":"public"},"exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js","require":"./dist/index.cjs","default":"./dist/index.js"}},"dependencies":{"@ai-sdk/provider":"3.0.1","ai":"6.0.6","ajv":"^8.17.1","decimal.js":"^10.4.3","zod":"^4.3.5","@ai-sdk-tool/middleware":"0.0.2"},"devDependencies":{"@types/node":"^25.0.3","tsup":"^8.5.1","typescript":"^5.9.3","@ai-sdkx/tsconfig":"0.0.1"},"scripts":{"build":"pnpm clean && tsup --tsconfig tsconfig.build.json","build:watch":"pnpm clean && tsup --watch --tsconfig tsconfig.build.json","clean":"rm -rf dist *.tsbuildinfo","typecheck":"tsc --noEmit","test:e2e":"tsup src/run-test.ts --format esm --clean --out-dir dist --no-config && node dist/run-test.js"},"_id":"@ai-sdk-tool/eval@1.1.0","bugs":{"url":"https://github.com/minpeter/ai-sdk-tool-call-middleware/issues"},"homepage":"https://github.com/minpeter/ai-sdk-tool-call-middleware#readme","_integrity":"sha512-UoTCJH0PMHrN/u6nQXZ6Tp0nBh5wwqwNPLreaLJm1f3tfAD/hr7FBHvhcXo2XpMAL9I7KHAqeIWAFLROllrGhg==","_resolved":"/tmp/e7860f058a32fad36fb7ec4d9bfe5b98/ai-sdk-tool-eval-1.1.0.tgz","_from":"file:ai-sdk-tool-eval-1.1.0.tgz","_nodeVersion":"24.12.0","_npmVersion":"11.6.2","dist":{"integrity":"sha512-UoTCJH0PMHrN/u6nQXZ6Tp0nBh5wwqwNPLreaLJm1f3tfAD/hr7FBHvhcXo2XpMAL9I7KHAqeIWAFLROllrGhg==","shasum":"a32286a0f507ea22c38a5cdce821e9a8c10c4774","tarball":"https://registry.npmjs.org/@ai-sdk-tool/eval/-/eval-1.1.0.tgz","fileCount":41,"unpackedSize":10492853,"attestations":{"url":"https://registry.npmjs.org/-/npm/v1/attestations/@ai-sdk-tool%2feval@1.1.0","provenance":{"predicateType":"https://slsa.dev/provenance/v1"}},"signatures":[{"keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U","sig":"MEQCIGHzomAre7EeUl88qiBE9dEvFMwi6KLy+7CHfw+Gz4gfAiBeklepJ6ApNJG5pkwY+D4cgkt/lLzVG0tqx4HFaPnQPA=="}]},"_npmUser":{"name":"GitHub Actions","email":"npm-oidc-no-reply@github.com","trustedPublisher":{"id":"github","oidcConfigId":"oidc:1d4deafb-0afd-43dc-8fce-d31ba032c66f"}},"directories":{},"maintainers":[{"name":"minpeter","email":"kali2005611@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages-npm-production","tmp":"tmp/eval_1.1.0_1767630615221_0.8327490449895811"},"_hasShrinkwrap":false}},"time":{"created":"2025-08-25T00:23:01.955Z","modified":"2026-01-05T16:30:15.798Z","0.1.1":"2025-08-25T00:23:02.296Z","0.1.2":"2025-08-27T13:59:50.467Z","0.1.3":"2025-08-29T03:30:35.425Z","0.1.4":"2025-08-30T14:20:00.996Z","0.1.5":"2025-08-31T08:07:14.504Z","0.1.6":"2025-09-01T03:19:20.324Z","0.1.7":"2025-09-02T14:56:38.882Z","0.1.8":"2025-09-04T04:17:03.702Z","1.0.0-canary.0":"2025-11-03T11:24:23.996Z","1.0.0-canary.1":"2025-12-26T09:46:12.206Z","1.0.0":"2026-01-02T14:27:33.443Z","1.1.0":"2026-01-05T16:30:15.423Z"},"bugs":{"url":"https://github.com/minpeter/ai-sdk-tool-call-middleware/issues"},"license":"Apache-2.0","homepage":"https://github.com/minpeter/ai-sdk-tool-call-middleware#readme","repository":{"type":"git","url":"git+https://github.com/minpeter/ai-sdk-tool-call-middleware.git","directory":"packages/eval"},"description":"Benchmarking and evaluation tools for AI SDK","maintainers":[{"name":"minpeter","email":"kali2005611@gmail.com"}],"readme":"# AI SDK - evaluation tool\n\n[![npm](https://img.shields.io/npm/v/@ai-sdk-tool/eval)](https://www.npmjs.com/package/@ai-sdk-tool/eval)\n[![npm](https://img.shields.io/npm/dt/@ai-sdk-tool/eval)](https://www.npmjs.com/package/@ai-sdk-tool/eval)\n\nThis package provides a standardized, extensible, and reproducible way to benchmark and evaluate the performance of Language Models (`LanguageModel` instances) within the Vercel AI SDK ecosystem.\n\nIt allows developers to:\n\n- Compare different models (e.g., Gemma, Llama, GPT) under the same conditions.\n- Quantify the impact of model updates or configuration changes.\n- Create custom benchmarks tailored to specific use cases (e.g., 'Korean proficiency', 'code generation').\n- Automate the evaluation process across a matrix of models and configurations.\n\n## Core Concepts\n\n- **Benchmark (`LanguageModelV3Benchmark`)**: A standardized interface for creating an evaluation task. It has a `run` method that takes a `LanguageModel` and returns a `BenchmarkResult`.\n- **`evaluate` function**: The core function that runs a set of benchmarks against one or more models and provides a report on the results.\n- **Reporter**: Formats the evaluation results into different outputs, such as a human-readable console report or a machine-readable JSON object.\n\n## Installation\n\n```bash\npnpm add @ai-sdk-tool/eval\n```\n\n## Quick Start\n\nHere's how to evaluate two different models against the built-in Berkeley Function-Calling Leaderboard (BFCL) benchmark for simple function calls.\n\n```typescript\nimport { evaluate, bfclSimpleBenchmark } from \"@ai-sdk-tool/eval\";\nimport { openrouter } from \"ai/providers/openrouter\";\n\n// 1. Define the models you want to evaluate\n\n// 2. Run the evaluation\nasync function runMyEvaluation() {\n  console.log(\"Starting model evaluation...\");\n\n  const results = await evaluate({\n    models: [/* your models here */],\n    benchmarks: [bfclSimpleBenchmark], // Use a built-in benchmark\n    reporter: \"console\", // 'console' or 'json'\n  });\n\n  console.log(\"Evaluation complete!\");\n  // The console reporter will have already printed a detailed report.\n}\n\nrunMyEvaluation();\n```\n\nRun the example from this repo:\n\n```bash\ncd examples/eval-core && pnpm dlx tsx src/bfcl-simple.ts\n```\n\n## Built-in Benchmarks\n\nThis package includes several pre-built benchmarks.\n\n- `bfclSimpleBenchmark`: Evaluates simple, single function calls.\n- `bfclParallelBenchmark`: Evaluates parallel (multi-tool) function calls.\n- `bfclMultipleBenchmark`: Evaluates multiple calls to the same function.\n- `bfclParallelMultipleBenchmark`: A combination of parallel and multiple function calls.\n- `bfclMultiTurnBaseBenchmark`: Evaluates BFCL v4 multi-turn base cases.\n- `bfclMultiTurnLongContextBenchmark`: Evaluates BFCL v4 multi-turn long-context cases.\n- `bfclMultiTurnMissFuncBenchmark`: Evaluates BFCL v4 multi-turn missing-function cases.\n- `bfclMultiTurnMissParamBenchmark`: Evaluates BFCL v4 multi-turn missing-parameter cases.\n- `jsonGenerationBenchmark`: Evaluates the model's ability to generate schema-compliant JSON.\n\nNote: Multi-turn benchmarks now execute tool calls with a native TypeScript implementation and do not require Python at runtime.\n\nBFCL evaluation data will be downloaded automatically on first run. For manual download, visit the [BFCL repository](https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard).\n\nTo try a JSON generation run locally:\n\n```bash\ncd examples/eval-core && pnpm dlx tsx src/json-generation.ts\n```\n\n## Creating a Custom Benchmark\n\nYou can easily create your own benchmark by implementing the `LanguageModelV3Benchmark` interface. This is useful for testing model performance on tasks specific to your application.\n\n**Example: A custom benchmark to test politeness.**\n\n```typescript\nimport {\n  LanguageModelV3Benchmark,\n  BenchmarkResult,\n  EvaluateOptions,\n} from \"@ai-sdk-tool/eval\";\nimport { LanguageModel, generateText } from \"ai\";\n\n// Define the benchmark object\nexport const politenessBenchmark: LanguageModelV3Benchmark = {\n  name: \"politeness-check\",\n  version: \"1.0.0\",\n  description: \"Checks if the model's response is polite.\",\n\n  async run(model: LanguageModel): Promise<BenchmarkResult> {\n    const { text } = await generateText({\n      model,\n      prompt:\n        \"A customer is angry because their order is late. Write a response.\",\n    });\n\n    const isPolite = !text.toLowerCase().includes(\"sorry, but\");\n    const score = isPolite ? 1 : 0;\n\n    return {\n      score,\n      success: isPolite,\n      metrics: {\n        length: text.length,\n      },\n      logs: [`Response: \"${text}\"`],\n    };\n  },\n};\n\n// You can then use it in the evaluate function:\n// await evaluate({\n//   models: myModel,\n//   benchmarks: [politenessBenchmark],\n// });\n```\n\n## License\n\nLicensed under Apache License 2.0. See the repository `LICENSE`. Include the `NOTICE` file in distributions.\n","readmeFilename":"README.md"}