{"_id":"@aroha-sdk/evals","_rev":"3-12b1307787475927dc4f46318a6cf9cc","name":"@aroha-sdk/evals","dist-tags":{"latest":"1.2.2"},"versions":{"1.1.0":{"name":"@aroha-sdk/evals","version":"1.1.0","keywords":["aroha","agent","evals","llm-as-judge","evaluation"],"license":"MIT","_id":"@aroha-sdk/evals@1.1.0","maintainers":[{"name":"aroha-labs","email":"projectmed99@gmail.com"}],"bin":{"aroha-evals":"dist/cli.js"},"dist":{"shasum":"b683be94edbbedf11b99bb72f9163d4172ce7526","tarball":"https://registry.npmjs.org/@aroha-sdk/evals/-/evals-1.1.0.tgz","fileCount":6,"integrity":"sha512-J1pUA0xxWpktCvDaNi6kNxN11slON7o0O6AKfd2O7iB/c2BCsQlep3G+/cWRBzSYNgDe7/8dAiamqP4OzGe7Bw==","signatures":[{"sig":"MEQCICZrk2if2GRtXLNN0ZVqdBUAHpXLeXHX3JKzJ2YUtLq1AiAB7LRbRCVDmZ+tyGsnCSqm08h5MqpJjXypmd1ZTX4wPA==","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":17439},"main":"./dist/index.js","type":"module","types":"./dist/index.d.ts","engines":{"node":">=22"},"exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js"}},"gitHead":"202398dcb7c1e33835ece99cdf19b9376cf884b3","scripts":{"dev":"tsc -p tsconfig.json --watch","test":"vitest run --passWithNoTests","build":"tsc -p tsconfig.json"},"_npmUser":{"name":"aroha-labs","email":"projectmed99@gmail.com"},"_npmVersion":"11.16.0","description":"Evaluation harness for Aroha agents — golden-dataset runner, LLM-as-judge, trajectory scoring","directories":{},"_nodeVersion":"24.18.0","_hasShrinkwrap":false,"devDependencies":{"vitest":"^1.6.0","typescript":"^5.4.5","@types/node":"^20.14.0"},"_npmOperationalInternal":{"tmp":"tmp/evals_1.1.0_1782995411142_0.17318054947037753","host":"s3://npm-registry-packages-npm-production"}},"1.1.1":{"name":"@aroha-sdk/evals","version":"1.1.1","keywords":["aroha","agent","evals","llm-as-judge","evaluation"],"license":"MIT","_id":"@aroha-sdk/evals@1.1.1","maintainers":[{"name":"aroha-labs","email":"projectmed99@gmail.com"}],"bin":{"aroha-evals":"dist/cli.js"},"dist":{"shasum":"82fc9323d78289d00c9908c831df753f70db92de","tarball":"https://registry.npmjs.org/@aroha-sdk/evals/-/evals-1.1.1.tgz","fileCount":31,"integrity":"sha512-KqUOE3TTyeIFoQGyUabS+UklYXMIk4uinxilTI6+WEuVpd9IWUUvS2B9X7XyVGt84Y279lkL01bcTnE4mosctA==","signatures":[{"sig":"MEUCIQDAfYCYqiBCsjRbxTsVKPZpaSfq+kpm/R+dIP57lCGxSwIgH5w+gzS0qHzn6nKq7aselBltZwTdnATldgJvIc3UdU4=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":79117},"main":"./dist/index.js","type":"module","types":"./dist/index.d.ts","engines":{"node":">=22"},"exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js"}},"gitHead":"a1d8b67f68730637c0c9c1935d1ecc3c6caa7078","scripts":{"dev":"tsc -p tsconfig.json --watch","test":"vitest run --passWithNoTests","build":"tsc -p tsconfig.json"},"_npmUser":{"name":"aroha-labs","email":"projectmed99@gmail.com"},"_npmVersion":"11.16.0","description":"Evaluation harness for Aroha agents — golden-dataset runner, LLM-as-judge, trajectory scoring","directories":{},"_nodeVersion":"24.18.0","_hasShrinkwrap":false,"devDependencies":{"vitest":"^1.6.0","typescript":"^5.4.5","@types/node":"^20.14.0"},"_npmOperationalInternal":{"tmp":"tmp/evals_1.1.1_1783112584399_0.895819810542624","host":"s3://npm-registry-packages-npm-production"}},"1.2.2":{"name":"@aroha-sdk/evals","version":"1.2.2","description":"Evaluation harness for Aroha agents — golden-dataset runner, LLM-as-judge, trajectory scoring","type":"module","main":"./dist/index.js","types":"./dist/index.d.ts","bin":{"aroha-evals":"dist/cli.js"},"exports":{".":{"import":"./dist/index.js","types":"./dist/index.d.ts"}},"scripts":{"build":"tsc -p tsconfig.json","test":"vitest run --passWithNoTests","dev":"tsc -p tsconfig.json --watch"},"engines":{"node":">=22"},"license":"MIT","keywords":["aroha","agent","evals","llm-as-judge","evaluation"],"devDependencies":{"typescript":"^5.4.5","vitest":"^1.6.0","@types/node":"^20.14.0"},"gitHead":"59f177e5c55a3a25977c1fd9895040f339806fb7","_id":"@aroha-sdk/evals@1.2.2","_nodeVersion":"24.18.0","_npmVersion":"11.16.0","dist":{"integrity":"sha512-3XnTie+Edee8eM3vdhPQAtAlh6UVRKLT7DadJErTwx1t+IVB05Kig1Jb0BBvLqKO3L8X8PIg45cJz2AFDRBdpg==","shasum":"99d40f40fa4737670f98be432b7bc2f76f2b8a6e","tarball":"https://registry.npmjs.org/@aroha-sdk/evals/-/evals-1.2.2.tgz","fileCount":32,"unpackedSize":81393,"signatures":[{"keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U","sig":"MEYCIQCYza1W+PFxPlj/8ybnGV5uIw7VHRbXH/gxwY8cqmS8RQIhAKpQDDWL6y7sJFK8rLFr/Mbf6A29xUyL/CP6azW2cZ+a"}]},"_npmUser":{"name":"aroha-labs","email":"projectmed99@gmail.com"},"directories":{},"maintainers":[{"name":"aroha-labs","email":"projectmed99@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages-npm-production","tmp":"tmp/evals_1.2.2_1783298875902_0.8674580883452983"},"_hasShrinkwrap":false}},"time":{"created":"2026-07-02T12:30:11.020Z","modified":"2026-07-06T00:47:56.147Z","1.1.0":"2026-07-02T12:30:11.288Z","1.1.1":"2026-07-03T21:03:04.543Z","1.2.2":"2026-07-06T00:47:56.047Z"},"license":"MIT","keywords":["aroha","agent","evals","llm-as-judge","evaluation"],"description":"Evaluation harness for Aroha agents — golden-dataset runner, LLM-as-judge, trajectory scoring","maintainers":[{"name":"aroha-labs","email":"projectmed99@gmail.com"}],"readme":"# @aroha-sdk/evals\n\nEvaluation harness for Aroha agents — verify mandate-respecting behavior across test datasets before shipping.\n\n## Install\n\n```bash\nnpm install -D @aroha-sdk/evals\n```\n\n## Quick example\n\n```typescript\nimport { runEvals, mandateEvalSuite } from \"@aroha-sdk/evals\";\nimport { issueIntentMandate } from \"@aroha-sdk/credentials\";\n\n// Built-in suite: tests that mandate bounds are respected\nconst results = await runEvals({\n  agentUrl: \"http://flight-agent/v1/run\",\n  issuerPublicKey: humanPublicKey,\n  suites: [\n    mandateEvalSuite({\n      // Tests that the agent rejects over-scope mandates\n      overScopeCases: [\n        {\n          mandate: await issueIntentMandate(humanDID, agentDID, { spendLimitUsd: 100 }, humanPrivateKey, 60_000),\n          action: { requestedSpend: 200 },\n          expectedOutcome: \"rejected\",\n        },\n      ],\n      // Tests that the agent accepts valid in-scope mandates\n      validCases: [\n        {\n          mandate: await issueIntentMandate(humanDID, agentDID, { spendLimitUsd: 300, allowedActions: [\"book-flights\"] }, humanPrivateKey, 60_000),\n          action: { requestedSpend: 299, action: \"book-flights\" },\n          expectedOutcome: \"accepted\",\n        },\n      ],\n      // Tests that forged mandates are rejected\n      forgedCases: [\n        {\n          forgedToken: \"eyJhbGciOi...tampered\",\n          expectedOutcome: \"rejected\",\n        },\n      ],\n    }),\n  ],\n});\n\nconsole.log(results.summary);\n// Passed: 12 / 12  Mandate violations: 0\n```\n\n## Why this exists in the mandate chain\n\nShipping a mandate-aware agent without evals risks silent regressions — an agent that passes valid mandates but also passes forged ones. `@aroha-sdk/evals` provides a structured harness that tests mandate acceptance, rejection, and forgery handling before any code ships.\n\n## API\n\n- `runEvals(opts)` — run one or more eval suites against an agent URL.\n- `mandateEvalSuite(opts)` — built-in suite covering over-scope rejection, valid mandate acceptance, forged mandate rejection, and TTL expiry.\n- `customEvalCase(opts)` — define a single eval case with a message, mandate, and expected outcome.\n- `EvalReport` — structured result with pass/fail counts, mandate violation count, and per-case details.\n\n## License\n\nMIT\n","readmeFilename":"README.md"}