"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __exportStar = (this && this.__exportStar) || function(m, exports) {
    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.SELECT_BEST_PROMPT = exports.TOOL_SELECTION_QUALITY_PROMPT = exports.AGENT_CUSTOM_EVAL_PROMPT = exports.GOAL_FULFILLMENT_PROMPT = exports.REASONING_COHERENCE_PROMPT = exports.EXECUTION_EFFICIENCY_PROMPT = exports.PLAN_ADHERENCE_EVAL_PROMPT = exports.PLAN_ADHERENCE_PROMPT = exports.SUGGEST_PROMPTS_SYSTEM_MESSAGE = exports.OPENAI_CLOSED_QA_PROMPT = exports.TRUSYS_FACTUALITY_PROMPT = exports.DEFAULT_GRADING_PROMPT = void 0;
const dedent_1 = __importDefault(require("dedent"));
/**
 * This module exports various prompts used for model graded evaluation.
 * They are imported and re-exported by the `prompts` module.
 *
 * @module prompts
 */
__exportStar(require("./external/ragas"), exports);
exports.DEFAULT_GRADING_PROMPT = JSON.stringify([
    {
        role: 'system',
        content: (0, dedent_1.default) `You are grading output according to a user-specified rubric. If the statement in the rubric is true, then the output passes the test. You respond with a JSON object with this structure: {reason: string, pass: boolean, score: number}

    Examples:

    <Output>Hello world</Output>
    <Rubric>Content contains a greeting</Rubric>
    {"reason": "the content contains the word 'Hello'", "pass": true, "score": 1.0}

    <Output>Avast ye swabs, repel the invaders!</Output>
    <Rubric>Does not speak like a pirate</Rubric>
    {"reason": "'avast ye' is a common pirate term", "pass": false, "score": 0.0}`,
    },
    {
        role: 'user',
        content: '<Output>\n{{ output }}\n</Output>\n<Rubric>\n{{ rubric }}\n</Rubric>',
    },
]);
// Based on https://github.com/openai/evals/blob/main/evals/registry/modelgraded/fact.yaml
// but modified for models capable of JSON output
exports.TRUSYS_FACTUALITY_PROMPT = JSON.stringify([
    {
        role: 'system',
        content: (0, dedent_1.default) `
      You are a precise factuality evaluator that compares a submitted answer to an expert answer. 

      Your task is to analyze the factual content while ignoring differences in style, grammar, or punctuation.
      You must categorize the submission into one of these options:

      (A) The submitted answer is a subset of the expert answer and is fully consistent with it.
      (B) The submitted answer is a superset of the expert answer and is fully consistent with it.
      (C) The submitted answer contains all the same details as the expert answer.
      (D) There is a disagreement between the submitted answer and the expert answer.
      (E) The answers differ, but these differences don't matter from the perspective of factuality.

      Respond ONLY with a JSON object in this format:
      {
        "category": "[LETTER]",
        "reason": "[DETAILED EXPLANATION]"
      }

      - The "category" must be a single letter A, B, C, D, or E.
      - Provide a clear, detailed explanation in the "reason" field.
      - Your response must be valid JSON with no additional text.`,
    },
    {
        role: 'user',
        content: (0, dedent_1.default) `
      I need you to compare these answers:

      <question>
      {{input}}
      </question>

      <expert_answer>
      {{ideal}}
      </expert_answer>

      <submitted_answer>
      {{completion}}
      </submitted_answer>

      Please analyze the factual relationship between these answers according to the categories you've been given.`,
    },
]);
exports.OPENAI_CLOSED_QA_PROMPT = JSON.stringify([
    {
        role: 'system',
        content: `You are assessing a submitted answer on a given task based on a criterion. Here is the data:
[BEGIN DATA]
***
[Task]: {{input}}
***
[Submission]: {{completion}}
***
[Criterion]: {{criteria}}
***
[END DATA]
Does the submission meet the criterion? First, write out in a step by step manner your reasoning about the criterion to be sure that your conclusion is correct. Avoid simply stating the correct answers at the outset. Then print only the single character "Y" or "N" (without quotes or punctuation) on its own line corresponding to the correct answer. At the end, repeat just the letter again by itself on a new line.

    Reasoning:`,
    },
]);
exports.SUGGEST_PROMPTS_SYSTEM_MESSAGE = {
    role: 'system',
    content: `You're helping a scientist who is tuning a prompt for a large language model.  You will receive messages, and each message is a full prompt.  Generate a candidate variation of the given prompt.  This variation will be tested for quality in order to select a winner.

Substantially revise the prompt, revising its structure and content however necessary to make it perform better, while preserving the original intent and including important details.

Your output is going to be copied directly into the program. It should contain the prompt ONLY`,
};
exports.PLAN_ADHERENCE_PROMPT = `You are an evaluation assistant. Your job is to evaluate plans generated by AI agents to determine whether it will accomplish a given user task based on the available tools or agents.

Here is the data:
    [BEGIN DATA]
    ************
    [User task]: {{ task }}
    ************
    [Tools or Agent definitions]: {{ definitions }}
    ************
    [Plan]: {{ plan }}
    [END DATA]

Definitions format: Either (1) a list of tools, each with "name", "description", and "arguments", or (2) a list of agent definitions, each with "name", "description", and "tool_definitions" (nested list of tools). Use whichever is provided to judge the plan.

Here is the criteria for evaluation
1. Does the plan include only valid and applicable tools/agents for the task?
2. Are the tools/agents used in the plan sufficient to accomplish the task?
3. Will the plan, as outlined, successfully achieve the desired outcome?
4. Is this the shortest and most efficient plan to accomplish the task?

Respond with a JSON object containing exactly two fields:
- "result": one of "ideal", "valid", or "invalid"
- "reason": a brief explanation for your evaluation

"ideal" means the plan generated is valid, uses only available tools/agents, is the shortest possible plan, and will likely accomplish the task.

"valid" means the plan generated is valid and uses only available tools/agents, but has doubts on whether it can successfully accomplish the task.

"invalid" means the plan generated includes invalid steps that cannot be used based on the available tools/agents.

Your response must be valid JSON only, with no other text or characters.`;
/** Evaluates how well the agent's tool_calls or agent_calls adhered to the stated plan. */
exports.PLAN_ADHERENCE_EVAL_PROMPT = `You are an evaluation assistant. Your job is to evaluate how well an AI agent's actual execution (tool calls or agent calls) adhered to its stated plan for a given user task.

Here is the data:
    [BEGIN DATA]
    ************
    [User task]: {{ task }}
    ************
    [Plan]: {{ plan }}
    ************
    [Actual tool calls or agent calls]: {{ trace }}
    [END DATA]

Trace format: Either (1) a list of tool calls, each with "name", "description", "arguments", and optionally "output", or (2) a list of agent calls, each with "name", "description", and "tool_calls" (nested list of tool calls). Use whichever is provided to compare against the plan.

Compare the plan to the trace. Then assign exactly one adherence level.

Respond with a JSON object containing exactly two fields:
- "result": one of "perfect", "strong", "partial", "weak", or "none"
- "reason": a brief explanation for your evaluation

Scoring criteria (use exactly these labels):

**1.0 — perfect**: Every planned step is explicitly and verifiably present in the trace, in correct order. No skipped or added steps. No ambiguity in matching.

**0.75 — strong**: All or nearly all steps are executed in order. At most one minor deviation (e.g., a trivial reordering or minor redundant step) that does not change the plan's structure.

**0.5 — partial**: Some steps clearly match, but others are missing, out of order, or replaced. At least one extra or ambiguous action appears. This is the highest score possible when there are any deviations.

**0.25 — weak**: Only a few steps from the plan appear in the trace, or multiple extraneous actions occur. The structure or sequence of the plan is mostly lost.

**0.0 — none**: The trace shows little or no resemblance to the plan. Steps are ignored, replaced, or executed in an entirely different order.

Your response must be valid JSON only, with no other text or characters.`;
/** Evaluates how efficiently the agent executed its steps. Score 0–3 returned in JSON. */
exports.EXECUTION_EFFICIENCY_PROMPT = `You are a meticulous and analytical EXECUTION EFFICIENCY evaluator. Your job is to score how efficiently an AI agent executed its steps to accomplish a user task.

Your assessment must strictly focus on the sequencing, resource utilization, and avoidance of redundant or wasteful actions within the execution itself, regardless of whether the task was ultimately successful or whether the plan was fully adhered to.

Here is the data:
    [BEGIN DATA]
    ************
    [User task]: {{ task }}
    ************
    [Execution trace (tool calls or agent calls)]: {{ trace }}
    ************
    [Agent output]: {{ agent_output }}
    [END DATA]

Assign a single integer score from 0 to 3 using the following criteria:

**3 — Highly efficient**: All relevant actions are executed exactly once, in a streamlined and optimized sequence. There is no unnecessary busy work, repetition, backtracking, or wasted computation. Each step genuinely contributes to progressing toward the goal without extraneous operations. Error handling is appropriately lean and resolves quickly without requiring multiple attempts due to easily correctable input errors. Verification steps provide unique feedback, serve as sanity checks, or use a demonstrably different approach from the initial approach to ensure correctness, without duplicating prior effort.

**2 — Mostly efficient**: Some minor instances of workflow inefficiency such as slightly redundant actions, non-ideal ordering of steps that cause minor rework, or missed opportunities for consolidation. There may be occasional minor input errors or misconfigurations that lead to a slightly increased number of attempts but are quickly corrected without major disruption. Inefficiencies have a noticeable but not significant impact.

**1 — Somewhat inefficient**: Multiple instances of workflow inefficiency such as redundant actions, non-ideal ordering of steps that cause rework, excessive error handling, missed opportunities for consolidation, or unnecessary resource use. There may be input errors or misconfigurations that lead to several repeated attempts that could have been avoided.

**0 — Highly inefficient**: Workflow is dominated by loops, duplicated efforts, poorly ordered sequence, or significant wasted computation that breaks progress. Multiple repeated tool or agent calls required to recover from preventable mistakes. Verification steps are highly redundant and provide no value. The workflow's operational flow is severely hampered by unnecessary or counterproductive actions.

For each step in the execution trace with an issue (e.g., redundancies, unnecessary retries, inefficient sequencing, missed optimization opportunities, or preventable errors), identify that step and explain the specific problem in your reason.

Respond with a JSON object containing exactly two fields:
- "score": an integer from 0 to 3
- "reason": a concise explanation identifying specific issues or confirming efficiency

Your response must be valid JSON only, with no other text or characters.`;
/** Evaluates logical consistency of the plan and execution. Score 0–3 returned in JSON. */
exports.REASONING_COHERENCE_PROMPT = `You are a meticulous and analytical LOGICAL CONSISTENCY evaluator. Your job is to score the logical consistency of an agentic system's plan and execution trace.

You must assign a single integer score from 0 to 3, where 0 is the lowest score according to the criteria and 3 is the highest possible score.

Here is the data:
    [BEGIN DATA]
    ************
    [User task]: {{ task }}
    ************
    [Plan]: {{ plan }}
    ************
    [Execution trace (tool calls or agent calls)]: {{ trace }}
    ************
    [Agent output]: {{ agent_output }}
    [END DATA]

Score the logical consistency of the trace, including both the plan and execution.

**3 — Fully consistent**: Every action, claim, and transition in the trace is explicitly justified using information available in the prior context. Each statement is directly supported by and traceable to previous data, instructions, or content; no part of the response is fabricated or inferred from unstated assumptions. If an error from an earlier step is identified and corrected, the error is explicitly acknowledged before the correction is made, maintaining logical transparency. Each system instruction is followed. The reasoning remains coherent and free of contradictions or logical leaps.

**2 — Mostly consistent**: There are occasional lapses in logic, minor unsupported assertions, or isolated explanatory gaps. Errors may be corrected, but corrections are occasionally introduced without clear acknowledgement of prior mistakes, creating minor inconsistencies or reducing transparency. Some statements may not be fully traceable to prior context, or some assumptions are made without explicit support from available evidence. Factual consistency may suffer from minor errors or embellishments, but the overall reasoning remains intact. Most previously assigned tasks and instructions remain intact.

**1 — Somewhat inconsistent**: More frequent lapses in logic, unsupported assertions, or contradictions. Corrections are sometimes made without acknowledgement. Some actions or facts are not clearly grounded in the given information. The reasoning has noticeable gaps or invalid assumptions.

**0 — Highly inconsistent**: There is frequent or severe breakdown in the logical flow; many statements are either unsupported by or cannot be grounded in the prior context. Corrections for earlier errors are often made without any explicit acknowledgement, resulting in contradictions or confusing transitions. Key actions or facts are invented, fabricated, or otherwise not observable in the given information. Major contradictions, invalid assumptions, or arbitrary transitions undermine the overall reasoning and conclusion. Most previously assigned tasks are not fulfilled, and internal system instructions are largely disregarded.

Be critical in your evaluation. For each step in the trace with an issue (e.g., contradictions, unsupported statements, or previous instructions not followed), identify that step and explain the problem specifically. Flag any implicit assumptions.

Respond with a JSON object containing exactly two fields:
- "score": an integer from 0 to 3
- "reason": a concise explanation identifying specific issues or confirming consistency

Your response must be valid JSON only, with no other text or characters.`;
/** Evaluates whether the agent's plan, execution, and output fulfilled the user's goal. */
exports.GOAL_FULFILLMENT_PROMPT = `You are an evaluation assistant. Your job is to determine whether an AI agent has fulfilled the user's goal based on the plan (if any), the execution trace (tool/agent calls), and the final output.

Here is the data:
    [BEGIN DATA]
    ************
    [User goal]: {{ task }}
    ************
    [Plan]: {{ plan }}
    ************
    [Execution trace (tool calls or agent calls)]: {{ trace }}
    ************
    [Agent output]: {{ agent_output }}
    [END DATA]

Evaluate:
1. Does the final output directly address and satisfy the user's original goal?
2. If a plan was provided, is it logically connected to achieving that goal? Did the execution follow it?
3. Do the tool/agent calls support progress toward the goal?
4. Is the output accurate, complete, and useful relative to the goal?
5. Are there any parts of the user's goal left unaddressed?

Respond with a JSON object containing exactly two fields:
- "result": one of "fully_fulfilled", "partially_fulfilled", or "not_fulfilled"
- "reason": a brief explanation for your evaluation

"fully_fulfilled" means the agent's output completely and accurately addresses the user's goal. The plan was sound and the output is a direct, useful answer.

"partially_fulfilled" means the agent's output addresses some aspects of the user's goal but is incomplete, has minor inaccuracies, or leaves notable parts unaddressed.

"not_fulfilled" means the agent's output fails to address the user's goal, is irrelevant, incorrect, or the plan was fundamentally flawed leading to an unhelpful result.

Your response must be valid JSON only, with no other text or characters.`;
/** Evaluates agent plan and execution against custom criteria (array of instructions). */
exports.AGENT_CUSTOM_EVAL_PROMPT = `You are a meticulous AGENT BEHAVIOUR evaluator. Your job is to judge whether an AI agent's plan and execution conform to a set of custom evaluation criteria.

Here is the data:
    [BEGIN DATA]
    ************
    [User task]: {{ task }}
    ************
    [Plan]: {{ plan }}
    ************
    [Agent/tool definitions]: {{ definitions }}
    ************
    [Execution trace (agent/tool calls)]: {{ trace }}
    ************
    [Agent output]: {{ agent_output }}
    ************
    [Evaluation criteria]:
    {{ criteria }}
    [END DATA]

Evaluate each criterion in order. For each criterion, decide whether the agent's plan and execution satisfy the criterion or violate it.
IMPORTANT: The judgement should be based purely on the criteria itself, not generally on the agents plan and execution.

Assign a single integer score from 0 to {{ max_score }} where:
- {{ max_score }}: Every applicable criterion is fully satisfied with no violations.
- Middle scores: Some criteria are satisfied; minor violations or gaps exist. Deduct one point per distinct criterion that is violated or only partially met.
- 0: Most applicable criteria are violated or cannot be verified.

For every criterion that is violated or not satisfied, identify the specific step or element in the trace/plan responsible and explain the problem.

Respond with a JSON object containing exactly two fields:
- "score": an integer from 0 to {{ max_score }}
- "reason": a concise explanation citing each violation or confirming compliance

Your response must be valid JSON only, with no other text or characters.`;
/** Evaluates whether the LLM selected the most appropriate tool(s) for the task. */
exports.TOOL_SELECTION_QUALITY_PROMPT = `You are an impartial judge evaluating an LLM's tool-calling behavior, specifically whether the LLM selected the most appropriate tool or tools for the task.

Your task: Determine whether the LLM's tool selection was correct or incorrect based on:
- The conversation context (input)
- The available tools
- The LLM's output and tool invocation(s)

Criteria:
Return "correct" only when ALL of the following are true:
- The LLM chose the best available tool for the user query OR correctly avoided tools if none were needed.
- The tool name exists in the available tools list.
- The tool is allowed and safe to call.
- The LLM selected the correct number of tools for the task.

Return "incorrect" if ANY of the following are true:
- The LLM used a hallucinated or nonexistent tool.
- The LLM selected a tool when none was needed.
- The LLM did not use a tool when one was required.
- The LLM chose a suboptimal or irrelevant tool.
- The LLM selected an unsafe or not-permitted tool.
- The tool name does not appear in the available tools list.

Before providing your final judgment, explain your reasoning and consider:
- What does the input context require?
- Can this be answered without tools, or is a tool necessary?
- If a tool was selected, does it exist in the available tools?
- Does the selected tool's description match the user's needs?
- Is the selection safe and appropriate?
- Is there a better tool available that should have been chosen instead?

Here is the data:
    [BEGIN DATA]
    ************
    [User task / input]: {{ input }}
    ************
    [Available tools]: {{ available_tools }}
    ************
    [Tool selection / invocations]: {{ tool_selection }}
    [END DATA]

Respond with a JSON object containing exactly two fields:
- "result": either "correct" or "incorrect"
- "reason": your reasoning and explanation

Your response must be valid JSON only, with no other text or characters.`;
exports.SELECT_BEST_PROMPT = JSON.stringify([
    {
        role: 'system',
        content: `You are comparing multiple pieces of text to see which best fits the following criteria: {{criteria}}

Here are the pieces of text:

{% for output in outputs %}
<Text index="{{ loop.index0 }}">
{{ output }}
</Text>
{% endfor %}

Output the index of the text that best fits the criteria. You must output a single integer.`,
    },
]);
//# sourceMappingURL=grading.js.map