{"_id":"@future-agi/ai-evaluation","_rev":"3-13e7c85603a425d775a56088755f0a19","name":"@future-agi/ai-evaluation","dist-tags":{"latest":"0.2.0"},"versions":{"0.1.0":{"name":"@future-agi/ai-evaluation","version":"0.1.0","keywords":["ai","evaluation","genai","machine-learning","typescript","model-evaluation","accuracy","esm","esnext"],"author":{"name":"Future AGI","email":"no-reply@futureagi.com"},"_id":"@future-agi/ai-evaluation@0.1.0","maintainers":[{"name":"sarthakfagi","email":"sarthak@futureagi.com"},{"name":"nvjkkartik","email":"n.v.j.kkartik@gmail.com"},{"name":"futureagi","email":"nikhil@futureagi.com"}],"homepage":"https://github.com/futureagi/ai-evaluation#readme","bugs":{"url":"https://github.com/futureagi/ai-evaluation/issues"},"dist":{"shasum":"dafc78ad3164e327a3620b0ee901ba7141f8aa55","tarball":"https://registry.npmjs.org/@future-agi/ai-evaluation/-/ai-evaluation-0.1.0.tgz","fileCount":114,"integrity":"sha512-vA5KFn2JKkD4E+z/sBSpSF5RW0m/lGz1gSFA5CEQziISpXFrHVNyRzdBEKVMdmNj5C/0c+U6i84BoehVB/YXGA==","signatures":[{"sig":"MEUCIBksyiDn2/4efuVkXUZnMkcWmt0g1AZLpBLQwXine17QAiEAjOxhmDnNuzmjxLYLjR48OJvjbvwz7A3q0hrfYiITVpY=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":703578},"jest":{"roots":["<rootDir>/src"],"preset":"ts-jest","testMatch":["**/__tests__/**/*.ts","**/?(*.)+(spec|test).ts","**/?(*.)+(integration.test).ts"],"testEnvironment":"node","collectCoverageFrom":["src/**/*.ts","!src/**/*.d.ts","!src/**/__tests__/**","!src/**/index.ts"]},"main":"./dist/src/index.js","type":"module","types":"./dist/src/index.d.ts","module":"./dist/esm/index.js","engines":{"node":">=18.0.0","pnpm":">=8.0.0"},"exports":{".":{"types":"./dist/src/index.d.ts","import":"./dist/esm/index.js","require":"./dist/src/index.js"}},"gitHead":"301e193864d1618d2bfb3d48bb496d2afc2ccbec","scripts":{"dev":"tsx src/index.ts","lint":"eslint . --ext .ts","test":"jest","build":"tsc --build tsconfig.json tsconfig.esm.json tsconfig.esnext.json && tsc-alias -p tsconfig.esm.json && pnpm run postbuild","clean":"rimraf dist","format":"prettier --write .","prepare":"pnpm run build","lint:fix":"eslint . --ext .ts --fix","prebuild":"rimraf dist","postbuild":"echo '{\"type\": \"module\"}' > ./dist/esm/package.json","typecheck":"tsc --noEmit","test:watch":"jest --watch","build:watch":"tsc --build --watch tsconfig.json","format:check":"prettier --check .","test:coverage":"jest --coverage"},"_npmUser":{"name":"nvjkkartik","email":"n.v.j.kkartik@gmail.com"},"repository":{"url":"git+https://github.com/futureagi/ai-evaluation.git","type":"git"},"_npmVersion":"10.9.2","description":"We help GenAI teams maintain high-accuracy for their Models in production.","directories":{},"_nodeVersion":"23.6.1","dependencies":{"axios":"^1.6.0","winston":"^3.11.0","@future-agi/sdk":">=0.1.0","@traceai/fi-core":">=0.1.15"},"publishConfig":{"access":"public"},"_hasShrinkwrap":false,"packageManager":"pnpm@8.15.0","devDependencies":{"tsx":"^4.0.0","jest":"^29.5.0","eslint":"^8.0.0","rimraf":"^5.0.0","ts-jest":"^29.1.0","prettier":"^3.0.0","tsc-alias":"^1.8.0","typescript":"^5.3.0","@types/jest":"^29.5.0","@types/node":"^20.0.0","eslint-config-prettier":"^9.0.0","eslint-plugin-prettier":"^5.0.0","@typescript-eslint/parser":"^7.0.0","@typescript-eslint/eslint-plugin":"^7.0.0"},"peerDependencies":{},"_npmOperationalInternal":{"tmp":"tmp/ai-evaluation_0.1.0_1753172252617_0.49844233411139505","host":"s3://npm-registry-packages-npm-production"}},"0.2.0":{"name":"@future-agi/ai-evaluation","version":"0.2.0","description":"We help GenAI teams maintain high-accuracy for their Models in production.","type":"module","main":"./dist/src/index.js","module":"./dist/esm/index.js","types":"./dist/src/index.d.ts","bin":{"fi":"dist/src/cli/main.js"},"exports":{".":{"import":"./dist/esm/index.js","require":"./dist/src/index.js","types":"./dist/src/index.d.ts"},"./local":{"import":"./dist/esm/local/index.js","require":"./dist/src/local/index.js","types":"./dist/src/local/index.d.ts"},"./cli":{"import":"./dist/esm/cli/main.js","require":"./dist/src/cli/main.js"}},"scripts":{"prebuild":"rimraf dist","build":"tsc --build tsconfig.json tsconfig.esm.json tsconfig.esnext.json && tsc-alias -p tsconfig.esm.json && pnpm run postbuild","postbuild":"echo '{\"type\": \"module\"}' > ./dist/esm/package.json","build:watch":"tsc --build --watch tsconfig.json","test":"jest --config jest.config.cjs","test:local":"jest --config jest.config.cjs --testPathPattern=local","test:watch":"jest --config jest.config.cjs --watch","test:coverage":"jest --config jest.config.cjs --coverage","lint":"eslint . --ext .ts","lint:fix":"eslint . --ext .ts --fix","format":"prettier --write .","format:check":"prettier --check .","clean":"rimraf dist","dev":"tsx src/index.ts","cli":"tsx src/cli/main.ts","typecheck":"tsc --noEmit","prepare":"pnpm run build"},"keywords":["ai","evaluation","genai","machine-learning","typescript","model-evaluation","accuracy","esm","esnext"],"author":{"name":"Future AGI","email":"no-reply@futureagi.com"},"repository":{"type":"git","url":"git+https://github.com/future-agi/ai-evaluation.git"},"bugs":{"url":"https://github.com/future-agi/ai-evaluation/issues"},"homepage":"https://github.com/future-agi/ai-evaluation#readme","engines":{"node":">=18.0.0","pnpm":">=8.0.0"},"packageManager":"pnpm@8.15.0","devDependencies":{"@types/jest":"^29.5.0","@types/node":"^20.0.0","@typescript-eslint/eslint-plugin":"^7.0.0","@typescript-eslint/parser":"^7.0.0","eslint":"^8.0.0","eslint-config-prettier":"^9.0.0","eslint-plugin-prettier":"^5.0.0","jest":"^29.5.0","prettier":"^3.0.0","rimraf":"^5.0.0","ts-jest":"^29.1.0","tsc-alias":"^1.8.0","tsx":"^4.0.0","typescript":"^5.3.0","@opentelemetry/api":"^1.9.0"},"peerDependencies":{"@opentelemetry/api":"^1.9.0"},"peerDependenciesMeta":{"@opentelemetry/api":{"optional":true}},"dependencies":{"@traceai/fi-core":"^0.1.16","axios":"^1.6.0","chalk":"^5.3.0","commander":"^12.0.0","ora":"^8.0.0","winston":"^3.11.0","yaml":"^2.3.0","zod":"^3.22.0"},"publishConfig":{"access":"public"},"gitHead":"5ca2b3aacd2b5277f479aa2e31341fae94196727","_id":"@future-agi/ai-evaluation@0.2.0","_nodeVersion":"20.20.2","_npmVersion":"11.13.0","dist":{"integrity":"sha512-hf1xpjuYGGnBM4BsDy7pqRXUhOoqcGqBascnviDRba0SwJyvSJxz+5o2I3FdMxFWMvS7/q3BDULPPL/DT5AAgw==","shasum":"5f333379b8fd0433df9dfca269911ad2a307f7bb","tarball":"https://registry.npmjs.org/@future-agi/ai-evaluation/-/ai-evaluation-0.2.0.tgz","fileCount":990,"unpackedSize":4161738,"attestations":{"url":"https://registry.npmjs.org/-/npm/v1/attestations/@future-agi%2fai-evaluation@0.2.0","provenance":{"predicateType":"https://slsa.dev/provenance/v1"}},"signatures":[{"keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U","sig":"MEUCIQClWkBH5JV8hkxVXBa5uRNn+08foqDFNS12v25kjhVh7QIgDe0us09Of+bLQU8kQ1ahpRJk5KazF2NzUEWSdeB97Qg="}]},"_npmUser":{"name":"GitHub Actions","email":"npm-oidc-no-reply@github.com","trustedPublisher":{"id":"github","oidcConfigId":"oidc:40bbd638-2989-4b6c-ae32-92d28191ad0d"}},"directories":{},"maintainers":[{"name":"sarthakfagi","email":"sarthak@futureagi.com"},{"name":"nvjkkartik","email":"n.v.j.kkartik@gmail.com"},{"name":"futureagi","email":"nikhil@futureagi.com"},{"name":"khushalsonawat","email":"khushalsonawat@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages-npm-production","tmp":"tmp/ai-evaluation_0.2.0_1777374756407_0.053950783141627934"},"_hasShrinkwrap":false}},"time":{"created":"2025-07-22T08:17:32.526Z","modified":"2026-04-28T11:12:36.966Z","0.1.0":"2025-07-22T08:17:32.804Z","0.2.0":"2026-04-28T11:12:36.611Z"},"bugs":{"url":"https://github.com/future-agi/ai-evaluation/issues"},"author":{"name":"Future AGI","email":"no-reply@futureagi.com"},"homepage":"https://github.com/future-agi/ai-evaluation#readme","keywords":["ai","evaluation","genai","machine-learning","typescript","model-evaluation","accuracy","esm","esnext"],"repository":{"type":"git","url":"git+https://github.com/future-agi/ai-evaluation.git"},"description":"We help GenAI teams maintain high-accuracy for their Models in production.","maintainers":[{"name":"sarthakfagi","email":"sarthak@futureagi.com"},{"name":"nvjkkartik","email":"n.v.j.kkartik@gmail.com"},{"name":"futureagi","email":"nikhil@futureagi.com"},{"name":"khushalsonawat","email":"khushalsonawat@gmail.com"}],"readme":"# @future-agi/ai-evaluation\n\nTypeScript SDK for Future AGI's AI evaluation platform. Evaluate LLM outputs with 50+ metrics including factual accuracy, groundedness, relevance, and more.\n\n## Installation\n\n```bash\nnpm install @future-agi/ai-evaluation\n# or\npnpm add @future-agi/ai-evaluation\n# or\nyarn add @future-agi/ai-evaluation\n```\n\n## Quick Start\n\n```typescript\nimport { evaluate } from '@future-agi/ai-evaluation';\n\n// Set your API key\nprocess.env.FI_API_KEY = 'your-api-key';\nprocess.env.FI_SECRET_KEY = 'your-secret-key';\n\n// Run an evaluation\nconst result = await evaluate(\n  'Factual Accuracy',\n  {\n    response: ['The capital of France is Paris.'],\n    context: ['Paris is the capital and largest city of France.']\n  }\n);\n\nconsole.log(result.eval_results[0].output); // Score: 0-1\nconsole.log(result.eval_results[0].reason); // Explanation\n```\n\n## Features\n\n- **Cloud Evaluations**: 50+ evaluation metrics via Future AGI API\n- **Local Evaluations**: Run heuristic metrics offline without API calls\n- **Hybrid Mode**: Automatically route between local and cloud execution\n- **Local LLM Support**: Use Ollama for LLM-as-judge evaluations locally\n- **Platform Integration**: Langfuse integration for observability\n- **Pipeline Evaluation**: Evaluate entire ML pipelines\n\n## Usage\n\n### Cloud Evaluation (Default)\n\n```typescript\nimport { Evaluator } from '@future-agi/ai-evaluation';\n\nconst evaluator = new Evaluator({\n  fiApiKey: 'your-api-key',\n  fiSecretKey: 'your-secret-key'\n});\n\n// Single evaluation\nconst result = await evaluator.evaluate(\n  'Groundedness',\n  {\n    query: ['What is machine learning?'],\n    response: ['Machine learning is a subset of AI...'],\n    context: ['Machine learning (ML) is a field of AI...']\n  },\n  { modelName: 'gpt-4o' }\n);\n\n// Async evaluation (returns immediately, poll for results)\nconst asyncResult = await evaluator.evaluate(\n  'Factual Accuracy',\n  { response: ['...'], context: ['...'] },\n  { isAsync: true }\n);\n\n// Get async result later\nconst finalResult = await evaluator.getEvalResult(asyncResult.eval_id);\n```\n\n### Local Evaluation (Offline)\n\nRun evaluations locally without API calls using heuristic metrics:\n\n```typescript\nimport { LocalEvaluator } from '@future-agi/ai-evaluation/local';\n\nconst evaluator = new LocalEvaluator();\n\n// String metrics\nconst containsResult = evaluator.evaluate(\n  'contains',\n  [{ response: 'Hello world' }],\n  { keyword: 'world' }\n);\n// Score: 1.0 (contains the keyword)\n\n// JSON validation\nconst jsonResult = evaluator.evaluate(\n  'json_schema',\n  [{ response: '{\"name\": \"John\", \"age\": 30}' }],\n  {\n    schema: {\n      type: 'object',\n      properties: { name: { type: 'string' }, age: { type: 'number' } },\n      required: ['name']\n    }\n  }\n);\n\n// Similarity metrics\nconst bleuResult = evaluator.evaluate(\n  'bleu_score',\n  [{ response: 'The cat sat on the mat' }],\n  { reference: 'The cat is on the mat' }\n);\n```\n\n### Available Local Metrics\n\n| Category | Metrics |\n|----------|---------|\n| **String** | `regex`, `contains`, `contains_all`, `contains_any`, `contains_none`, `one_line`, `equals`, `starts_with`, `ends_with`, `length_less_than`, `length_greater_than`, `length_between` |\n| **JSON** | `contains_json`, `is_json`, `json_schema` |\n| **Similarity** | `bleu_score`, `rouge_score`, `recall_score`, `levenshtein_similarity`, `numeric_similarity`, `semantic_list_contains` |\n\n### Hybrid Evaluation\n\nAutomatically route between local and cloud execution:\n\n```typescript\nimport { HybridEvaluator, OllamaLLM } from '@future-agi/ai-evaluation/local';\nimport { Evaluator } from '@future-agi/ai-evaluation';\n\n// Setup hybrid evaluator with local LLM\nconst localLLM = new OllamaLLM({ model: 'llama3.2' });\nconst cloudEvaluator = new Evaluator();\n\nconst hybrid = new HybridEvaluator({\n  localLLM,\n  cloudEvaluator,\n  preferLocal: true,      // Prefer local when possible\n  fallbackToCloud: true,  // Fall back to cloud if local fails\n  offlineMode: false      // Set true to disable cloud entirely\n});\n\n// Heuristic metrics run locally\nconst localResult = await hybrid.evaluate(\n  'contains',\n  [{ response: 'Hello world' }],\n  { keyword: 'world' }\n);\n\n// LLM-based metrics use local Ollama if available\nconst llmResult = await hybrid.evaluate(\n  'groundedness',\n  [{\n    query: 'What is AI?',\n    response: 'AI is artificial intelligence.',\n    context: 'Artificial intelligence (AI) is...'\n  }]\n);\n```\n\n### Local LLM with Ollama\n\nUse Ollama for local LLM-as-judge evaluations:\n\n```typescript\nimport { OllamaLLM } from '@future-agi/ai-evaluation/local';\n\n// Ensure Ollama is running: ollama serve\nconst llm = new OllamaLLM({\n  model: 'llama3.2',           // Model name\n  baseUrl: 'http://localhost:11434',  // Ollama URL\n  temperature: 0.0,            // Deterministic output\n  maxTokens: 1024,\n  timeout: 120                 // Seconds\n});\n\n// Check availability\nconst isAvailable = await llm.isAvailable();\n\n// Direct generation\nconst response = await llm.generate('Explain quantum computing');\n\n// Chat completion\nconst chatResponse = await llm.chat([\n  { role: 'system', content: 'You are a helpful assistant.' },\n  { role: 'user', content: 'What is machine learning?' }\n]);\n\n// LLM-as-judge\nconst judgment = await llm.judge(\n  'What is the capital of France?',      // Query\n  'The capital of France is Paris.',     // Response to evaluate\n  'Evaluate factual accuracy. Score 0-1.', // Criteria\n  'Paris is the capital of France.'      // Optional context\n);\n// Returns: { score: 1.0, passed: true, reason: '...' }\n```\n\n### Pipeline Evaluation\n\nEvaluate entire ML pipelines:\n\n```typescript\nconst evaluator = new Evaluator();\n\n// Submit pipeline evaluation\nawait evaluator.evaluatePipeline(\n  'my-project',\n  'v1.0.0',\n  [\n    { input: 'query1', output: 'response1', context: 'ctx1' },\n    { input: 'query2', output: 'response2', context: 'ctx2' }\n  ]\n);\n\n// Get results for multiple versions\nconst results = await evaluator.getPipelineResults(\n  'my-project',\n  ['v1.0.0', 'v1.1.0', 'v2.0.0']\n);\n```\n\n### Langfuse Integration\n\nEnable observability with Langfuse:\n\n```typescript\nconst evaluator = new Evaluator({\n  fiApiKey: process.env.FI_API_KEY,\n  fiSecretKey: process.env.FI_SECRET_KEY,\n  langfuseSecretKey: process.env.LANGFUSE_SECRET_KEY,\n  langfusePublicKey: process.env.LANGFUSE_PUBLIC_KEY,\n  langfuseHost: process.env.LANGFUSE_HOST\n});\n\n// Evaluations will be logged to Langfuse\nconst result = await evaluator.evaluate(\n  'Groundedness',\n  { response: ['...'], context: ['...'] },\n  { platform: 'langfuse', customEvalName: 'my-eval' }\n);\n```\n\n## API Reference\n\n### Main Exports (`@future-agi/ai-evaluation`)\n\n| Export | Description |\n|--------|-------------|\n| `Evaluator` | Main class for cloud evaluations |\n| `evaluate()` | Convenience function for single evaluation |\n| `list_evaluations()` | List available evaluation templates |\n| `get_eval_result()` | Get async evaluation result |\n| `evaluate_pipeline()` | Evaluate a pipeline |\n| `get_pipeline_results()` | Get pipeline results |\n\n### Local Exports (`@future-agi/ai-evaluation/local`)\n\n| Export | Description |\n|--------|-------------|\n| `LocalEvaluator` | Run heuristic metrics locally |\n| `HybridEvaluator` | Route between local and cloud |\n| `OllamaLLM` | Local LLM client via Ollama |\n| `LocalLLMFactory` | Factory for creating LLM instances |\n| `canRunLocally()` | Check if metric runs locally |\n| `requiresLLM()` | Check if metric needs LLM |\n| Individual metrics | `contains`, `regex`, `bleuScore`, etc. |\n\n## Environment Variables\n\n| Variable | Description |\n|----------|-------------|\n| `FI_API_KEY` | Future AGI API key |\n| `FI_SECRET_KEY` | Future AGI secret key |\n| `FI_BASE_URL` | API base URL (optional) |\n| `LANGFUSE_SECRET_KEY` | Langfuse secret key (optional) |\n| `LANGFUSE_PUBLIC_KEY` | Langfuse public key (optional) |\n| `LANGFUSE_HOST` | Langfuse host URL (optional) |\n\n## Requirements\n\n- Node.js >= 18.0.0\n- For local LLM: [Ollama](https://ollama.ai/) installed and running\n\n## License\n\nMIT\n\n## Links\n\n- [Documentation](https://docs.futureagi.com)\n- [GitHub](https://github.com/futureagi/ai-evaluation)\n- [Issues](https://github.com/futureagi/ai-evaluation/issues)\n","readmeFilename":"README.md"}