{"_id":"@cyanheads/evals-mcp-server","_rev":"3-61e2b80915e1e9731d284d49e3ba0686","name":"@cyanheads/evals-mcp-server","dist-tags":{"latest":"0.1.4"},"versions":{"0.1.2":{"name":"@cyanheads/evals-mcp-server","version":"0.1.2","keywords":["mcp","mcp-server","model-context-protocol","typescript","bun","stdio","streamable-http","evals","evaluation","grader","rlvr","verifiable-rewards","inspect-ai","lm-eval-harness","ai-agent"],"author":"cyanheads (https://github.com/cyanheads/evals-mcp-server#readme)","license":"Apache-2.0","_id":"@cyanheads/evals-mcp-server@0.1.2","maintainers":[{"name":"cyanheads","email":"casey@caseyjhand.com"}],"homepage":"https://github.com/cyanheads/evals-mcp-server#readme","bugs":{"url":"https://github.com/cyanheads/evals-mcp-server/issues"},"bin":{"evals-mcp-server":"dist/index.js"},"dist":{"shasum":"e69d260e23827675acb1130a4ad7376748850c94","tarball":"https://registry.npmjs.org/@cyanheads/evals-mcp-server/-/evals-mcp-server-0.1.2.tgz","fileCount":82,"integrity":"sha512-8taJMxTtYZwg/WuFsC09M+xh/y7EaGM2m/smN91fcIiDmbrOj4aA9UBWq3R/O2scTQvhhywD+nh8SR2zkynhfg==","signatures":[{"sig":"MEUCIQCRjwn3cEve3xWVFUGuYDfTiUWbMuv+lQk4zkXyS+XzXAIgCXXO/3dgGBeRLKgzNTRqFuHwoU5D6qTBE/jtU61Okr0=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":445769},"main":"dist/index.js","type":"module","types":"dist/index.d.ts","shasum":"e69d260e23827675acb1130a4ad7376748850c94","engines":{"bun":">=1.3.0","node":">=24.0.0"},"funding":[{"url":"https://github.com/sponsors/cyanheads","type":"github"},{"url":"https://www.buymeacoffee.com/cyanheads","type":"buy_me_a_coffee"}],"mcpName":"io.github.cyanheads/evals-mcp-server","scripts":{"test":"vitest run","tree":"bun run scripts/tree.ts","build":"bun run scripts/build.ts","clean":"bun run scripts/clean.ts","start":"node dist/index.js","bundle":"bun run build && npx -y @anthropic-ai/mcpb pack . dist/evals-mcp-server.mcpb && bun run scripts/clean-mcpb.ts dist/evals-mcp-server.mcpb","format":"biome check --write .","rebuild":"bun run scripts/clean.ts && bun run scripts/build.ts","devcheck":"bun run scripts/devcheck.ts","lint:mcp":"bun run scripts/lint-mcp.ts","start:http":"MCP_TRANSPORT_TYPE=http node dist/index.js","list-skills":"bun run scripts/list-skills.ts","publish-mcp":"mcp-publisher login github -token \"$(security find-generic-password -a \"$USER\" -s mcp-publisher-github-pat -w)\" && mcp-publisher publish","start:stdio":"MCP_TRANSPORT_TYPE=stdio node dist/index.js","audit:refresh":"rm -f bun.lock && bun install && bun audit","format:unsafe":"biome check --write --unsafe .","lint:packaging":"bun run scripts/lint-packaging.ts","release:github":"bun run scripts/release-github.ts","changelog:build":"bun run scripts/build-changelog.ts","changelog:check":"bun run scripts/build-changelog.ts --check"},"_npmUser":{"name":"cyanheads","email":"casey@caseyjhand.com"},"_integrity":"sha512-8taJMxTtYZwg/WuFsC09M+xh/y7EaGM2m/smN91fcIiDmbrOj4aA9UBWq3R/O2scTQvhhywD+nh8SR2zkynhfg==","repository":{"url":"git+https://github.com/cyanheads/evals-mcp-server.git","type":"git"},"_npmVersion":"10.8.3","description":"Author verifiable eval records through a draft → review → revise → submit loop with server-enforced graders; compile to JSONL/CSV/Inspect/lm-eval via MCP. STDIO or Streamable HTTP.","directories":{},"_nodeVersion":"24.3.0","dependencies":{"ajv":"^8.20.0","zod":"^4.4.3","mathjs":"^15.2.0","nanoid":"^5.1.16","pino-pretty":"^13.1.3","@cyanheads/mcp-ts-core":"^0.10.9"},"publishConfig":{"access":"public"},"_hasShrinkwrap":false,"packageManager":"bun@1.3.2","devDependencies":{"ignore":"^7.0.5","vitest":"^4.1.8","depcheck":"^1.4.7","tsc-alias":"^1.8.17","typescript":"^6.0.3","@types/node":"^26.0.1","@biomejs/biome":"^2.5.1"},"_npmOperationalInternal":{"tmp":"tmp/evals-mcp-server_0.1.2_1782622745569_0.6683535493332287","host":"s3://npm-registry-packages-npm-production"}},"0.1.3":{"name":"@cyanheads/evals-mcp-server","version":"0.1.3","keywords":["mcp","mcp-server","model-context-protocol","typescript","bun","stdio","streamable-http","evals","evaluation","grader","rlvr","verifiable-rewards","inspect-ai","lm-eval-harness","ai-agent"],"author":"Casey Hand (https://caseyjhand.com)","license":"Apache-2.0","_id":"@cyanheads/evals-mcp-server@0.1.3","maintainers":[{"name":"cyanheads","email":"casey@caseyjhand.com"}],"homepage":"https://github.com/cyanheads/evals-mcp-server#readme","bugs":{"url":"https://github.com/cyanheads/evals-mcp-server/issues"},"bin":{"evals-mcp-server":"dist/index.js"},"dist":{"shasum":"2ef3eb5e81514e7bad5c7dda175003c5f46a90e0","tarball":"https://registry.npmjs.org/@cyanheads/evals-mcp-server/-/evals-mcp-server-0.1.3.tgz","fileCount":83,"integrity":"sha512-tD0MtQanuiZOyQ+S3xELC5273xQfBnK0mfqJ6S+y3QxAs/dXV0n+qJjDKQ3p/uR4w5XTYBeNFY8e57JsVHN43A==","signatures":[{"sig":"MEUCIEEog93S+OtBpjI+4jKGrp3HTDQIrkWZb4Rkzku2Lx8pAiEAsUh0BzITQcpA0NA+htyxRIic7XaO1xj9ycXtVabE5Xk=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":451626},"main":"dist/index.js","type":"module","types":"dist/index.d.ts","shasum":"2ef3eb5e81514e7bad5c7dda175003c5f46a90e0","engines":{"bun":">=1.3.0","node":">=24.0.0"},"funding":[{"url":"https://github.com/sponsors/cyanheads","type":"github"},{"url":"https://www.buymeacoffee.com/cyanheads","type":"buy_me_a_coffee"}],"mcpName":"io.github.cyanheads/evals-mcp-server","scripts":{"test":"vitest run","tree":"bun run scripts/tree.ts","build":"bun run scripts/build.ts","clean":"bun run scripts/clean.ts","start":"node dist/index.js","bundle":"bun run build && npx -y @anthropic-ai/mcpb pack . dist/evals-mcp-server.mcpb && bun run scripts/clean-mcpb.ts dist/evals-mcp-server.mcpb","format":"biome check --write .","rebuild":"bun run scripts/clean.ts && bun run scripts/build.ts","devcheck":"bun run scripts/devcheck.ts","lint:mcp":"bun run scripts/lint-mcp.ts","start:http":"MCP_TRANSPORT_TYPE=http node dist/index.js","list-skills":"bun run scripts/list-skills.ts","publish-mcp":"mcp-publisher login github -token \"$(security find-generic-password -a \"$USER\" -s mcp-publisher-github-pat -w)\" && mcp-publisher publish","start:stdio":"MCP_TRANSPORT_TYPE=stdio node dist/index.js","audit:refresh":"rm -f bun.lock && bun install && bun audit","format:unsafe":"biome check --write --unsafe .","lint:packaging":"bun run scripts/lint-packaging.ts","release:github":"bun run scripts/release-github.ts","changelog:build":"bun run scripts/build-changelog.ts","changelog:check":"bun run scripts/build-changelog.ts --check"},"_npmUser":{"name":"cyanheads","email":"casey@caseyjhand.com"},"_integrity":"sha512-tD0MtQanuiZOyQ+S3xELC5273xQfBnK0mfqJ6S+y3QxAs/dXV0n+qJjDKQ3p/uR4w5XTYBeNFY8e57JsVHN43A==","repository":{"url":"git+https://github.com/cyanheads/evals-mcp-server.git","type":"git"},"_npmVersion":"10.8.3","description":"Author verifiable eval records through a draft → review → revise → submit loop with server-enforced graders; compile to JSONL/CSV/Inspect/lm-eval via MCP. STDIO or Streamable HTTP.","directories":{},"_nodeVersion":"26.3.0","dependencies":{"ajv":"^8.20.0","zod":"^4.4.3","mathjs":"^15.2.0","nanoid":"^6.0.1","pino-pretty":"^13.1.3","@cyanheads/mcp-ts-core":"^0.12.3"},"publishConfig":{"access":"public"},"_hasShrinkwrap":false,"packageManager":"bun@1.4.0","devDependencies":{"ignore":"^7.0.6","vitest":"^4.1.11","depcheck":"^1.4.7","tsc-alias":"^1.9.2","typescript":"^7.0.2","@types/node":"26.1.1","@biomejs/biome":"^2.5.9","@socketsecurity/bun-security-scanner":"^1.1.2"},"_npmOperationalInternal":{"tmp":"tmp/evals-mcp-server_0.1.3_1787427990603_0.584958640639522","host":"s3://npm-registry-packages-npm-production"}},"0.1.4":{"_id":"@cyanheads/evals-mcp-server@0.1.4","bin":{"evals-mcp-server":"dist/index.js"},"bugs":{"url":"https://github.com/cyanheads/evals-mcp-server/issues"},"dist":{"shasum":"4ac52a76e2343c93d8d81bcdf03d2c641c893f0f","tarball":"https://registry.npmjs.org/@cyanheads/evals-mcp-server/-/evals-mcp-server-0.1.4.tgz","fileCount":84,"integrity":"sha512-aX6LLVZ08vkAs+vgwwlLuym/yn6XjWLtfvcntm2c30EOaMXusqJY/boVb8ShxVVyjEXn5JB60fJtXt4WHH2WLA==","signatures":[{"sig":"MEYCIQCtSBOq6t+1n22+QBmM2aK5TIMZxKlMfgGthTAKa3pFoQIhAOZykwi8+CF3ZjnsNKnI0rWe9yD9QXkBmI0W0Y5pEGzY","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"},{"keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U","sig":"MEYCIQCh8c5EvcLL3Q4erv6pkQMjH1qtNYc3GfjRZFHIEZHi+AIhAP6sn9knaNBi+gz2hWKOo//t+zv1mlB0YMbhcX49CeYf"}],"unpackedSize":470308},"main":"dist/index.js","name":"@cyanheads/evals-mcp-server","type":"module","types":"dist/index.d.ts","author":"Casey Hand (https://caseyjhand.com)","shasum":"4ac52a76e2343c93d8d81bcdf03d2c641c893f0f","engines":{"bun":">=1.4.0","node":">=24.0.0"},"funding":[{"url":"https://github.com/sponsors/cyanheads","type":"github"},{"url":"https://www.buymeacoffee.com/cyanheads","type":"buy_me_a_coffee"}],"license":"Apache-2.0","mcpName":"io.github.cyanheads/evals-mcp-server","scripts":{"test":"vitest run","tree":"bun run scripts/tree.ts","build":"bun run scripts/build.ts","clean":"bun run scripts/clean.ts","start":"node dist/index.js","bundle":"bun run build && npx -y @anthropic-ai/mcpb pack . dist/evals-mcp-server.mcpb && bun run scripts/clean-mcpb.ts dist/evals-mcp-server.mcpb","format":"biome check --write .","rebuild":"bun run scripts/clean.ts && bun run scripts/build.ts","devcheck":"bun run scripts/devcheck.ts","lint:mcp":"bun run scripts/lint-mcp.ts","audit:fix":"bun audit fix","start:http":"MCP_TRANSPORT_TYPE=http node dist/index.js","list-skills":"bun run scripts/list-skills.ts","publish-mcp":"mcp-publisher login github -token \"$(security find-generic-password -a \"$USER\" -s mcp-publisher-github-pat -w)\" && mcp-publisher publish","start:stdio":"MCP_TRANSPORT_TYPE=stdio node dist/index.js","audit:refresh":"rm -f bun.lock && bun install && bun audit","format:unsafe":"biome check --write --unsafe .","lint:packaging":"bun run scripts/lint-packaging.ts","release:github":"bun run scripts/release-github.ts","changelog:build":"bun run scripts/build-changelog.ts","changelog:check":"bun run scripts/build-changelog.ts --check"},"version":"0.1.4","_npmUser":{"name":"cyanheads","email":"casey@caseyjhand.com"},"homepage":"https://github.com/cyanheads/evals-mcp-server#readme","keywords":["mcp","mcp-server","model-context-protocol","typescript","bun","stdio","streamable-http","evals","evaluation","grader","rlvr","verifiable-rewards","inspect-ai","lm-eval-harness","ai-agent"],"_integrity":"sha512-aX6LLVZ08vkAs+vgwwlLuym/yn6XjWLtfvcntm2c30EOaMXusqJY/boVb8ShxVVyjEXn5JB60fJtXt4WHH2WLA==","repository":{"url":"git+https://github.com/cyanheads/evals-mcp-server.git","type":"git"},"_npmVersion":"10.8.3","description":"Author verifiable eval records through a draft → review → revise → submit loop with server-enforced graders; compile to JSONL/CSV/Inspect/lm-eval via MCP. STDIO or Streamable HTTP.","directories":{},"maintainers":[{"name":"cyanheads","email":"casey@caseyjhand.com"}],"_nodeVersion":"26.3.0","dependencies":{"ajv":"^8.20.0","zod":"^4.6.5","mathjs":"^15.2.0","nanoid":"^6.0.1","pino-pretty":"^13.1.3","@cyanheads/mcp-ts-core":"^0.13.6"},"publishConfig":{"access":"public"},"_hasShrinkwrap":false,"packageManager":"bun@1.4.0","devDependencies":{"ignore":"^7.0.9","vitest":"^5.0.1","depcheck":"^1.4.7","tsc-alias":"^1.9.5","typescript":"^7.0.2","@types/node":"26.6.2","@biomejs/biome":"^2.5.14","@socketsecurity/bun-security-scanner":"^1.1.2"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages-npm-production","tmp":"tmp/evals-mcp-server_0.1.4_1790051455079_0.0751975010965058"}}},"time":{"created":"2026-06-28T04:59:05.397Z","modified":"2026-09-22T04:30:55.348Z","0.1.2":"2026-06-28T04:59:05.708Z","0.1.3":"2026-08-22T19:46:30.739Z","0.1.4":"2026-09-22T04:30:55.181Z"},"bugs":{"url":"https://github.com/cyanheads/evals-mcp-server/issues"},"author":"Casey Hand (https://caseyjhand.com)","license":"Apache-2.0","homepage":"https://github.com/cyanheads/evals-mcp-server#readme","keywords":["mcp","mcp-server","model-context-protocol","typescript","bun","stdio","streamable-http","evals","evaluation","grader","rlvr","verifiable-rewards","inspect-ai","lm-eval-harness","ai-agent"],"repository":{"url":"git+https://github.com/cyanheads/evals-mcp-server.git","type":"git"},"description":"Author verifiable eval records through a draft → review → revise → submit loop with server-enforced graders; compile to JSONL/CSV/Inspect/lm-eval via MCP. STDIO or Streamable HTTP.","maintainers":[{"name":"cyanheads","email":"casey@caseyjhand.com"}],"readme":"
\n

@cyanheads/evals-mcp-server

\n

Author verifiable eval records through a draft → review → revise → submit loop with server-enforced graders; compile to JSONL/CSV/Inspect/lm-eval via MCP. STDIO or Streamable HTTP.\n

9 Tools • 1 Resource
\n

\n
\n\n
\n\n[![Version](https://img.shields.io/badge/Version-0.1.4-blue.svg?style=flat-square)](./CHANGELOG.md) [![License](https://img.shields.io/badge/License-Apache%202.0-orange.svg?style=flat-square)](./LICENSE) [![Docker](https://img.shields.io/badge/Docker-ghcr.io-2496ED?style=flat-square&logo=docker&logoColor=white)](https://github.com/users/cyanheads/packages/container/package/evals-mcp-server) [![MCP SDK](https://img.shields.io/badge/MCP%20SDK-^2.0.0-green.svg?style=flat-square)](https://modelcontextprotocol.io/) [![npm](https://img.shields.io/npm/v/@cyanheads/evals-mcp-server?style=flat-square&logo=npm&logoColor=white)](https://www.npmjs.com/package/@cyanheads/evals-mcp-server) [![TypeScript](https://img.shields.io/badge/TypeScript-^7.0.2-3178C6.svg?style=flat-square)](https://www.typescriptlang.org/) [![Bun](https://img.shields.io/badge/Bun-v1.4.0-blueviolet.svg?style=flat-square)](https://bun.sh/)\n\n
\n\n
\n\n[![Install in Claude Desktop](https://img.shields.io/badge/Install_in-Claude_Desktop-D97757?style=for-the-badge&logo=anthropic&logoColor=white)](https://github.com/cyanheads/evals-mcp-server/releases/latest/download/evals-mcp-server.mcpb) [![Install in Cursor](https://cursor.com/deeplink/mcp-install-dark.svg)](https://cursor.com/en/install-mcp?name=evals-mcp-server&config=eyJjb21tYW5kIjoibnB4IiwiYXJncyI6WyIteSIsIkBjeWFuaGVhZHMvZXZhbHMtbWNwLXNlcnZlciJdfQ==) [![Install in VS Code](https://img.shields.io/badge/VS_Code-Install_Server-0098FF?style=for-the-badge&logo=visualstudiocode&logoColor=white)](https://vscode.dev/redirect?url=vscode:mcp/install?%7B%22name%22%3A%22evals-mcp-server%22%2C%22command%22%3A%22npx%22%2C%22args%22%3A%5B%22-y%22%2C%22%40cyanheads%2Fevals-mcp-server%22%5D%7D)\n\n[![Framework](https://img.shields.io/badge/Built%20on-@cyanheads/mcp--ts--core-67E8F9?style=flat-square)](https://www.npmjs.com/package/@cyanheads/mcp-ts-core)\n\n
\n\n---\n\n## Overview\n\nVerifiable eval records, authored through a draft → review → surgical-revise → submit loop with server-enforced graders. Create a draft carrying its own executable grader, patch it surgically field by field, and submit through a committability gate that requires the gold to pass, a declared negative case to fail, and an independent verification to agree — then compile submitted records to JSONL, CSV, Inspect AI, or lm-evaluation-harness. Runs as a stdio process or a local Streamable HTTP server.\n\n### Tools\n\n| Tool | Description |\n|:---|:---|\n| `evals_describe_schema` | Return the required and optional fields plus grader options for a task type. Call before drafting. |\n| `evals_create_draft` | Create a draft eval record carrying its own grader; returns the parsed record, a review protocol, and a verification subagent prompt. |\n| `evals_get_record` | Read a draft or submitted record by id; the id is stable across submit. |\n| `evals_revise_draft` | Apply a surgical `set` / `append` / `unset` patch to a draft by dotted path; re-runs the self-consistency check. |\n| `evals_discard_draft` | Delete a draft record by id. Draft-only. |\n| `evals_run_check` | Run a grader spec against candidate answers and get PASS/REJECT per candidate, decoupled from any saved record. |\n| `evals_submit_draft` | Finalize a draft through the committability gate, then freeze it. |\n| `evals_list_records` | Browse and filter records by status, domain, task type, or tag. Returns a compact summary per record. |\n| `evals_export_records` | Compile submitted records to JSONL, CSV, Inspect AI, or lm-evaluation-harness and write the artifact under `exports/`. |\n\n### Resources\n\n| Resource | Description |\n|:---|:---|\n| `eval://record/{id}` | A single draft or submitted record by id — the same payload `evals_get_record` returns, for resource-capable clients. |\n\nAll record data is also reachable through the tool surface — `evals_get_record` for a single record, `evals_list_records` to browse. The resource is a convenience mirror for clients that support resources, not the access path.\n\n## Capability reference\n\n### `evals_describe_schema` tool\n\n- Static — derived from the record and grader Zod schemas, no disk or runtime state\n- `task_type` is one of `numeric`, `exact_answer`, `set_answer`, `mcq`, `regex_answer`, `json_answer`, `free_response`\n- Returns the gold shape, applicable grader kind(s), required/optional fields, and per-type authoring notes (e.g. `mcq` needs `choices`, `free_response` needs an `llm_rubric` grader)\n\n---\n\n### `evals_create_draft` tool\n\n- Validates against the `task_type` discriminated union and persists the draft; `mcq` requires `choices`, `free_response` requires an `llm_rubric` grader\n- Runs a self-consistency check — the grader must PASS against `gold` and each `discrimination.positive`, and REJECT each `discrimination.negative`\n- Returns the normalized record, a per-field review protocol, a ready-to-paste verification-subagent prompt, and what's still required before submit\n- Accepts optional draft-time `verification` evidence and `captures` (EvalsIDs) when provenance is already in hand\n- Typed errors: `grader_unexecutable`, `task_type_constraint`, `mcq_choice_mismatch`\n- Stays `draft` — passing self-consistency proves the grader discriminates, not that the gold is correct\n\n---\n\n### `evals_get_record` tool\n\n- Reads by `id`, stable across submit — resolves whether the record is still a draft or already submitted\n- Returns the full record, including its grader, discrimination cases, and verification evidence\n- `not_found` when no record matches; recovery points to `evals_list_records`\n\n---\n\n### `evals_revise_draft` tool\n\n- Explicit `set` (dotted-path → value), `append` (dotted-path → array items), and `unset` (dotted paths) operations — never a full-record rewrite\n- Cannot target `task_type` or server-owned fields — start a new draft to change the discriminant\n- Re-validates the full record shape and per-task-type constraints after the patch, and re-runs self-consistency since the grader may have moved\n- Returns the updated record and an itemized `changed` list (op, path, before, after)\n- Draft-only — `record_frozen` on a submitted id\n- Typed errors: `not_found`, `record_frozen`, `invalid_patch_path`, `task_type_constraint`, `mcq_choice_mismatch`\n\n---\n\n### `evals_discard_draft` tool\n\n- Deletes a draft record by `draft_id`\n- Draft-only — `record_frozen` when the id refers to a submitted record\n- A missing id reports `not_found` rather than a distinct \"already discarded\" error — effectively idempotent\n\n---\n\n### `evals_run_check` tool\n\n- Runs a grader spec against one or more `candidates` (strings, numbers, objects, or arrays) without touching a saved record\n- Returns PASS/REJECT and a `detail` per candidate, plus the `resolved` comparison value (e.g. the math.js-evaluated numeric target)\n- `gold` applies only to gold-relative kinds (`exact_match`); it's a no-op for target-embedding kinds like `numeric` and `mcq`\n- `llm_rubric` cannot run here — submission relies on recorded independent verification instead\n- Typed errors: `grader_unexecutable`, `mcq_choice_mismatch`\n\n---\n\n### `evals_submit_draft` tool\n\n- The committability gate: the gold must PASS its grader, ≥1 declared negative must be REJECTED, and a recorded, decorrelated independent verification must agree with the gold\n- Resolves and embeds any `captures` from `EVALS_CAPTURE_DIR`, cross-checking the gold against the authoritative captured value\n- Rejects duplicates by `content_hash`; `confirm` (or `EVALS_REQUIRE_CONFIRMATION`) can require human confirmation through multi-round input before finalizing\n- On pass, flips the record to `submitted`, stamps `submitted_at` and a `checksum`, and freezes it; otherwise refuses and the record stays a draft\n- `free_response` is admitted on recorded independent verification alone and flagged `server_verified: false`\n- Typed errors: `not_found`, `record_frozen`, `verification_incomplete`, `grader_failed_on_gold`, `verification_disagrees_with_gold`, `missing_negative_case`, `negative_case_passed`, `duplicate`, `decorrelation_violation`, `capture_unresolved`, `submit_declined`\n\n---\n\n### `evals_list_records` tool\n\n- Filters by `status` (draft/submitted), `domain`, `task_type`, or `tag`; up to 500 per call (default 50)\n- Returns a compact summary per record (id, status, task_type, domain, tags, timestamps), newest-first — not full records\n- Discloses truncation (`shown`, `cap`, total count) when the limit is hit, so a partial set is never mistaken for the whole corpus\n\n---\n\n### `evals_export_records` tool\n\n- Formats: `jsonl` (lossless), `csv` (flattened, lossy summary), `inspect` (UK AISI Inspect AI), `lm-eval` (EleutherAI lm-evaluation-harness)\n- Optional `domain` / `task_type` / `tag` filter\n- Only `submitted` records are exported — drafts are skipped\n- Writes the artifact under `exports/` and returns its path, record count, byte size, and a short preview instead of dumping it inline\n\n---\n\n### `eval://record/{id}` resource\n\n- Returns the same payload as `evals_get_record`, as `application/json`\n- `id` comes from `evals_list_records` or a draft/submit response\n- `not_found` when no record matches\n\n## Features\n\nBuilt on [`@cyanheads/mcp-ts-core`](https://github.com/cyanheads/mcp-ts-core): stdio and Streamable HTTP transports, pluggable auth (`none` / `jwt` / `oauth`), swappable storage (`in-memory`, `filesystem`, `Supabase`, `Cloudflare KV/R2/D1`), structured logging with optional OpenTelemetry tracing.\n\nEval authoring:\n\n- A `draft → review → surgical-revise → submit` loop, with the server acting as both scribe (normalize, persist, compile) and adversarial checker (runs the record's own grader, rejects what doesn't hold up)\n- Records are a Zod `discriminatedUnion` on `task_type` — `numeric`, `exact_answer`, `set_answer`, `mcq`, `regex_answer`, `json_answer`, `free_response`\n- A typed grader DSL serialized with each record — deterministic kinds (`numeric` via math.js, `exact_match`, `set_match`, `regex`, `mcq`, `json_match`) run server-side; `llm_rubric` relies on recorded independent verification\n- An enforced committability gate at submit: the gold must pass its own grader, ≥1 negative must be rejected, and a recorded decorrelated verification must agree with the gold\n- Plain JSON files under `EVALS_DATA_DIR` — inspectable, diffable, version-controllable records, with drafts, submitted records, and exports kept separate\n\nAgent-friendly output:\n\n- Instructional responses — `evals_create_draft` and `evals_revise_draft` return the parsed record parroted back, a per-field review protocol, and a ready-to-paste verification-subagent prompt\n- Self-consistency verdicts — every draft/revise response reports per-positive and per-negative pass/reject results, not just a boolean\n- Truncation disclosure — `evals_list_records` reports `shown` / `cap` / total count when the limit is hit, so a partial set is never mistaken for the whole corpus\n- Typed refusal — the submit gate fails with a typed `reason` plus a recovery hint, so a rejected record tells the agent exactly what to fix\n\n## Getting started\n\nAdd the following to your MCP client configuration file. Set `EVALS_DATA_DIR` to a writable folder — the server manages `drafts/`, `submitted/`, and `exports/` under it.\n\n```json\n{\n \"mcpServers\": {\n \"evals-mcp-server\": {\n \"type\": \"stdio\",\n \"command\": \"bunx\",\n \"args\": [\"@cyanheads/evals-mcp-server@latest\"],\n \"env\": {\n \"MCP_TRANSPORT_TYPE\": \"stdio\",\n \"MCP_LOG_LEVEL\": \"info\",\n \"EVALS_DATA_DIR\": \"/absolute/path/to/evals-data\"\n }\n }\n }\n}\n```\n\nOr with npx (no Bun required):\n\n```json\n{\n \"mcpServers\": {\n \"evals-mcp-server\": {\n \"type\": \"stdio\",\n \"command\": \"npx\",\n \"args\": [\"-y\", \"@cyanheads/evals-mcp-server@latest\"],\n \"env\": {\n \"MCP_TRANSPORT_TYPE\": \"stdio\",\n \"MCP_LOG_LEVEL\": \"info\",\n \"EVALS_DATA_DIR\": \"/absolute/path/to/evals-data\"\n }\n }\n }\n}\n```\n\nOr with Docker:\n\n```json\n{\n \"mcpServers\": {\n \"evals-mcp-server\": {\n \"type\": \"stdio\",\n \"command\": \"docker\",\n \"args\": [\n \"run\", \"-i\", \"--rm\",\n \"-e\", \"MCP_TRANSPORT_TYPE=stdio\",\n \"-e\", \"EVALS_DATA_DIR=/data\",\n \"-v\", \"evals-data:/data\",\n \"ghcr.io/cyanheads/evals-mcp-server:latest\"\n ]\n }\n }\n}\n```\n\nFor Streamable HTTP, set the transport and start the server:\n\n```sh\nMCP_TRANSPORT_TYPE=http MCP_HTTP_PORT=3010 EVALS_DATA_DIR=./evals-data bun run start:http\n# Server listens at http://localhost:3010/mcp\n```\n\n### Prerequisites\n\n- [Bun v1.4.0](https://bun.sh/) or higher (or Node.js v24+).\n- A writable directory for `EVALS_DATA_DIR`. No external API key is required.\n\n### Installation\n\n1. **Clone the repository:**\n\n```sh\ngit clone https://github.com/cyanheads/evals-mcp-server.git\n```\n\n2. **Navigate into the directory:**\n\n```sh\ncd evals-mcp-server\n```\n\n3. **Install dependencies:**\n\n```sh\nbun install\n```\n\n4. **Configure environment:**\n\n```sh\ncp .env.example .env\n# edit .env and set EVALS_DATA_DIR\n```\n\n## Configuration\n\nAll server configuration is validated at startup via Zod schemas in `src/config/server-config.ts`.\n\n| Variable | Description | Default |\n|:---|:---|:---|\n| `EVALS_DATA_DIR` | Root folder for record JSON; the store manages `drafts/`, `submitted/`, and `exports/` under it. | `./evals-data` |\n| `EVALS_REQUIRE_CONFIRMATION` | When `true`, `evals_submit_draft` requests human confirmation through multi-round input before finalizing. | `false` |\n| `EVALS_DEFAULT_LICENSE` | Default `metadata.license` applied when a draft omits one (e.g. `CC-BY-4.0`). | — |\n| `EVALS_CAPTURE_DIR` | Directory of framework-written tool-call captures; when set, `captures` EvalsIDs resolve to full dumps. | — |\n| `MCP_TRANSPORT_TYPE` | Transport: `stdio` or `http`. | `stdio` |\n| `MCP_HTTP_PORT` | Port for the HTTP server. | `3010` |\n| `MCP_SESSION_MODE` | HTTP session mode: `auto`, `stateful`, or `stateless` (`auto` resolves to `stateful`). A `stateless` HTTP start is refused, since a 2025-era client can answer the `evals_submit_draft` confirmation only over a live session. No effect on stdio. | `stateful` |\n| `MCP_AUTH_MODE` | Auth mode: `none`, `jwt`, or `oauth`. | `none` |\n| `MCP_LOG_LEVEL` | Log level (RFC 5424). | `info` |\n| `OTEL_ENABLED` | Enable [OpenTelemetry instrumentation](https://github.com/cyanheads/mcp-ts-core/tree/main/docs/telemetry). | `false` |\n\nSee [`.env.example`](./.env.example) for the full list of optional overrides.\n\n## Running the server\n\n### Local development\n\n- **Build and run:**\n\n ```sh\n # One-time build\n bun run rebuild\n\n # Run the built server\n bun run start:stdio\n # or\n bun run start:http\n ```\n\n- **Run checks and tests:**\n\n ```sh\n bun run devcheck # Lint, format, typecheck, security\n bun run test # Vitest test suite\n bun run lint:mcp # Validate MCP definitions against spec\n ```\n\n### Docker\n\n```sh\ndocker build -t evals-mcp-server .\ndocker run --rm -e MCP_TRANSPORT_TYPE=stdio -e EVALS_DATA_DIR=/data -v evals-data:/data evals-mcp-server\n```\n\nThe Dockerfile defaults to HTTP transport, stateful session mode, and logs to `/var/log/evals-mcp-server`. OpenTelemetry peer dependencies are installed by default — build with `--build-arg OTEL_ENABLED=false` to omit them.\n\n## Project structure\n\n| Directory | Purpose |\n|:---|:---|\n| `src/index.ts` | `createApp()` entry point — registers tools and the resource, inits the record-store and exporter services. |\n| `src/config` | Server-specific environment variable parsing and validation with Zod. |\n| `src/mcp-server/tools` | Tool definitions (`*.tool.ts`). |\n| `src/mcp-server/resources` | Resource definitions (`*.resource.ts`). |\n| `src/services/eval-record` | The record schema, draft builder, and submit gate. |\n| `src/services/grader` | Deterministic grader DSL execution and the committability check. |\n| `src/services/record-store` | On-disk JSON record CRUD, the draft→submitted move, and export writes. |\n| `src/services/exporter` | Compiling submitted records to JSONL/CSV/Inspect/lm-eval. |\n| `tests/` | Unit and integration tests mirroring `src/`. |\n\n## Development guide\n\nSee [`CLAUDE.md`/`AGENTS.md`](./CLAUDE.md) for development guidelines and architectural rules. The short version:\n\n- Handlers throw, framework catches — no `try/catch` in tool logic\n- Use `ctx.log` for request-scoped logging; records persist to disk via the `record-store` service, not `ctx.state`\n- Register new tools and resources in the `createApp()` arrays in `src/index.ts`\n- The server is the source of truth — validate inputs, run the grader as a hard gate, and never admit a record on assertion alone\n\n## Contributing\n\nIssues are welcome. Run checks and tests before submitting:\n\n```sh\nbun run devcheck\nbun run test\n```\n\n## License\n\nApache-2.0 — see [LICENSE](LICENSE) for details.\n","readmeFilename":"README.md"}