import{toErrorMessage}from"#shared/errors.js";import{scopeEvalTargetHandle}from"#evals/target.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{resolveArtifactDirectory,writeArtifacts}from"#evals/runner/artifacts.js";import{executeCase}from"#evals/runner/execute-case.js";import{findUnmetRequirements,formatUnmetRequirements,resolveCaseRequirements}from"#evals/requirements.js";import{computeCaseVerdict}from"#evals/runner/verdict.js";async function executeEval(e){let{evaluation:t,target:n,reporters:a,appRoot:o,client:s}=e,c=new Date().toISOString();for(let e of a)await e.onEvalStart(t,n);let l=await t.load(),u=t.maxConcurrency??8,d=[],f=[...l],p=new Set,m=Promise.resolve();for(;f.length>0||p.size>0;){for(;f.length>0&&p.size<u;){let r=f.shift();if(r===void 0)break;let i=(async()=>{let i=await executeSingleCase({client:s,evaluation:t,failOnSkip:e.failOnSkip===!0,onCaseLog:e.onCaseLog,target:n,testCase:r});d.push(i),m=m.then(async()=>{for(let e of a)await e.onCaseComplete(i)})})().finally(()=>{p.delete(i)});p.add(i)}p.size>0&&await Promise.race(p)}await m;let h=new Date().toISOString(),g={id:t.id,target:n,cases:d,startedAt:c,completedAt:h,passed:countVerdicts(d,`passed`),failed:countVerdicts(d,`failed`),scored:countVerdicts(d,`scored`),skipped:countVerdicts(d,`skipped`),errored:d.filter(e=>e.error!==void 0&&e.skipReason===void 0).length};await writeArtifacts(resolveArtifactDirectory(o,t.id),g);for(let e of a)await e.onEvalComplete(g);return g}async function executeSingleCase(r){let{client:i,evaluation:s,failOnSkip:u,onCaseLog:d,target:f,testCase:p}=r,m=resolveCaseRequirements(s,p),h=findUnmetRequirements(m,f);if(h.length>0)return createRequirementResult(p,h,{failOnSkip:u});let g,_;try{g=await executeCase({client:i,onLog:d===void 0?void 0:e=>d(p.id,e),requirements:m,target:f,testCase:p,task:s.task,timeoutMs:s.timeoutMs})}catch(t){_=toErrorMessage(t),g={output:null,finalMessage:null,status:`failed`,events:[],derived:createEmptyDerivedFacts()}}let v=scopeEvalTargetHandle(f,{requirements:m}),y=await runChecks(s,p,g,v),b=await runScorers(s,p,g),x=computeCaseVerdict({error:_,checks:y,scores:b,thresholds:s.thresholds});return{case:p,result:g,checks:y,scores:b,verdict:x,error:_}}async function runChecks(t,n,r,i){let a=[...t.checks??[],...n.checks??[]];if(a.length===0)return[];let o={case:n,result:r,target:i},s=[];for(let[t,n]of a.entries())try{s.push(await n(o))}catch(r){s.push({name:n.name===``?`check#${t}`:n.name,passed:!1,message:toErrorMessage(r)})}return s}function createRequirementResult(e,t,n){let r=formatUnmetRequirements(t);return{case:e,result:createEmptyTaskResult(),checks:[],scores:[],verdict:n.failOnSkip?`failed`:`skipped`,skipReason:r}}function createEmptyTaskResult(){return{output:null,finalMessage:null,status:`completed`,events:[],derived:createEmptyDerivedFacts()}}async function runScorers(e,t,n){let r={input:caseInputToString(t),output:String(n.output??``),expected:t.expected===void 0?void 0:String(t.expected),model:e.model,modelOptions:e.modelOptions,case:t,result:n},i=[];for(let n of[...e.scores,...t.scores??[]]){let e=await n(r);e!==null&&i.push(e)}return i}function caseInputToString(e){return`input`in e?typeof e.input==`string`?e.input:JSON.stringify(e.input):``}function countVerdicts(e,t){return e.filter(e=>e.verdict===t).length}export{executeEval};