"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.inferProviderResponseRequirements = inferProviderResponseRequirements;
const TEXT_ONLY = {
    guidance: 'Uses the primary model/application output string (after transformResponse). Typical ProviderResponse.output only; no special audio/video/trace fields required.',
};
const AGENT_TRACE = {
    guidance: 'Expects agent/tool trace data: JSON output or vars containing tool_calls / agent_calls, plan, and definitions—often under output.trusys or merged vars from trace parsing. Typical JSON paths: trusys.agent_calls, trusys.tool_calls, trusys.plan, trusys.tool_definitions, trusys.agent_definitions. Configure transformResponse (or an agent provider) so the evaluated output exposes that structure, or supply the fields via test vars.',
};
const RAG_CONTEXT_VARS = {
    guidance: 'Needs retrieval grounding: tests[].vars.query and tests[].vars.context (or assertion.context), or output JSON with trusys.query / trusys.context. ProviderResponse.output is still the graded string/object. Populate vars from your dataset or map KB results in transformResponse into the shape the evaluator reads.',
};
const VIDEO = {
    guidance: 'Needs a video asset: ProviderResponse.video.url is preferred, or a stable public video URL in the text output so the metric can fetch frames. ProviderResponse fields involved: output, video.',
};
const AUDIO = {
    guidance: 'Needs audio: ProviderResponse.audio (transcript and/or audio data) or metadata used for transcript-style metrics; fields often include output, audio, metadata. Phone providers populate audio; HTTP apps may need a custom mapping per product docs.',
};
const MULTIMODAL = {
    guidance: 'Expects multimodal references: text plus image URLs or structured multimodal parts in output or vars so the metric can compare against references.',
};
const FINISH_REASON = {
    guidance: 'Reads ProviderResponse.finishReason compared to the expected value; ensure the provider exposes finishReason when using this metric.',
};
const COST = {
    guidance: 'Requires ProviderResponse.cost. Many custom HTTP apps omit cost—use managed LLM providers or extend the provider to attach usage cost before evaluation.',
};
const LATENCY = {
    guidance: 'Uses evaluator-measured latencyMs; cached runs may not include latency—rerun with --noCache when this metric must pass.',
};
const PERPLEXITY = {
    guidance: 'Requires logprobs on ProviderResponse (logProbs). Enable token-level probabilities on the model provider; most bare HTTP templates do not expose this.',
};
const OPENAI_TOOLS = {
    guidance: 'Expects OpenAI-style tools/function call objects in the evaluated JSON output so validators can parse tool_calls / function_call structure.',
};
const BY_NAME = {
    'finish-reason': FINISH_REASON,
    cost: COST,
    latency: LATENCY,
    perplexity: PERPLEXITY,
    'perplexity-score': PERPLEXITY,
    hallucination: {
        guidance: 'Needs non-empty grounding context: assertion.context, tests[].vars.context, or output.trusys.context, plus the model output text. Same retrieval wiring as other context-grounded metrics.',
    },
    'answer-relevance': {
        guidance: 'Uses the prompt (or tests[].vars.query when set) and the string output; optional query in tests[].vars overrides the prompt text for grading.',
    },
    'conversation-relevancy': {
        guidance: 'Requires tests[].vars.context (required in the UI) plus conversational output so relevancy can be judged against that context.',
    },
    'is-valid-openai-function-call': OPENAI_TOOLS,
    'is-valid-openai-tools-call': OPENAI_TOOLS,
    'tool-correctness': {
        guidance: `${AGENT_TRACE.guidance} Also set the assertion tools field (expected tool names).`,
    },
};
const RAG_CONTEXT_METRICS = new Set([
    'context-faithfulness',
    'context-recall',
    'context-relevance',
]);
function negatedBase(name) {
    return name.startsWith('not-') ? name.slice(4) : name;
}
function inferProviderResponseRequirements(name, group) {
    const special = BY_NAME[name];
    if (special) {
        return special;
    }
    const baseName = negatedBase(name);
    if (baseName !== name && BY_NAME[baseName]) {
        return BY_NAME[baseName];
    }
    if (group === 'agent-metrics') {
        return AGENT_TRACE;
    }
    if (RAG_CONTEXT_METRICS.has(name) || RAG_CONTEXT_METRICS.has(baseName)) {
        return RAG_CONTEXT_VARS;
    }
    if (group === 'video-analysis' || name === 'text-to-video') {
        return VIDEO;
    }
    if (group === 'audio-analysis') {
        return AUDIO;
    }
    if (group === 'multimodal') {
        return MULTIMODAL;
    }
    if (group === 'code-evaluation') {
        return {
            guidance: 'Treats output as source code (string). Language or style may come from assertion additionalFields or inference.',
        };
    }
    if (group === 'conversational-evaluation') {
        return {
            guidance: 'Expects conversational transcript or turns in output or vars (shape varies by metric). JSON hints that help: messages, conversation.',
        };
    }
    if (name === 'moderation' || name === 'guardrails') {
        return {
            guidance: 'Sends model output (and sometimes input) to moderation or guardrail models; plain text output is usually sufficient.',
        };
    }
    if (group === 'metrics' && (name.includes('rouge') || name === 'meteor' || name === 'bleu' || name === 'gleu')) {
        return {
            guidance: 'Compares output string to reference text from assertion value or vars; ensure the reference side is populated for meaningful scores.',
        };
    }
    return TEXT_ONLY;
}
//# sourceMappingURL=inferProviderResponseRequirements.js.map