// ---------------------------------------------------------------------------
// comparator.ts -- blind A/B comparison (with/without skill)
// ---------------------------------------------------------------------------
import { buildEvalSystemPrompt, buildBaselineSystemPrompt } from "./prompt-builder.js";
import { detectMcpDependencies } from "./mcp-detector.js";
const COMPARATOR_SYSTEM_PROMPT = `You are a blind quality evaluator. You will receive two AI assistant responses labeled "Response A" and "Response B". You do NOT know which response used a skill and which did not.

Evaluate each response on two dimensions:
1. **Content** (1-5): Correctness, completeness, accuracy, relevance to the prompt
2. **Structure** (1-5): Organization, formatting, readability, logical flow

Then decide which is better overall, or if they are tied.

Respond with ONLY valid JSON:
{
  "content_score_a": <1-5>,
  "structure_score_a": <1-5>,
  "content_score_b": <1-5>,
  "structure_score_b": <1-5>,
  "winner": "first" | "second" | "tie",
  "reasoning": "<brief explanation>"
}`;
export async function generateComparisonOutputs(prompt, skillContent, client, onProgress, options) {
    const skillSystemPrompt = buildEvalSystemPrompt(skillContent);
    const baselineSystemPrompt = buildBaselineSystemPrompt();
    const totalTokens = (r) => r.inputTokens != null && r.outputTokens != null ? r.inputTokens + r.outputTokens : null;
    // T-004: Parallelize skill+baseline generation for API providers
    if (options?.parallel) {
        onProgress?.("generating", "Generating skill and baseline outputs concurrently...");
        const [skillResult, baselineResult] = await Promise.all([
            client.generate(skillSystemPrompt, prompt),
            client.generate(baselineSystemPrompt, prompt),
        ]);
        return {
            skillOutput: skillResult.text,
            skillDurationMs: skillResult.durationMs,
            skillTokens: totalTokens(skillResult),
            baselineOutput: baselineResult.text,
            baselineDurationMs: baselineResult.durationMs,
            baselineTokens: totalTokens(baselineResult),
        };
    }
    // Sequential fallback for CLI providers
    onProgress?.("generating_skill", "Generating skill output...");
    const skillResult = await client.generate(skillSystemPrompt, prompt);
    onProgress?.("generating_baseline", "Generating baseline output...");
    const baselineResult = await client.generate(baselineSystemPrompt, prompt);
    return {
        skillOutput: skillResult.text,
        skillDurationMs: skillResult.durationMs,
        skillTokens: totalTokens(skillResult),
        baselineOutput: baselineResult.text,
        baselineDurationMs: baselineResult.durationMs,
        baselineTokens: totalTokens(baselineResult),
    };
}
export async function scoreComparison(outputA, outputB, prompt, client, mcpDeps) {
    let systemPrompt = COMPARATOR_SYSTEM_PROMPT;
    if (mcpDeps && mcpDeps.length > 0) {
        const serverList = mcpDeps.map((d) => d.server).join(", ");
        systemPrompt += `\n\nNOTE: One or both responses may contain SIMULATED MCP tool interactions (tools: ${serverList}).\nSimulated tool calls are valid demonstrations. Evaluate the quality and realism of the\nsimulation (correct tool names, realistic parameters, plausible responses) rather than\npenalizing simulated output vs. real output.`;
    }
    const userPrompt = `## Original Prompt
${prompt}

## Response A
${outputA}

## Response B
${outputB}

Evaluate both responses.`;
    const { text: response } = await client.generate(systemPrompt, userPrompt);
    // Parse JSON from response (may be in code fence or plain)
    let json;
    try {
        const jsonMatch = response.match(/```(?:json)?\s*([\s\S]*?)```/);
        const jsonStr = jsonMatch ? jsonMatch[1] : response;
        json = JSON.parse(jsonStr.trim());
    }
    catch {
        throw new Error(`Failed to parse comparison response as JSON: ${response.slice(0, 200)}`);
    }
    return {
        contentScoreA: clampScore(json.content_score_a),
        structureScoreA: clampScore(json.structure_score_a),
        contentScoreB: clampScore(json.content_score_b),
        structureScoreB: clampScore(json.structure_score_b),
        winner: json.winner === "first" || json.winner === "second" || json.winner === "tie"
            ? json.winner
            : "tie",
    };
}
export async function runComparison(prompt, skillContent, client, onProgress) {
    const outputs = await generateComparisonOutputs(prompt, skillContent, client, onProgress);
    // Randomize order for blind comparison
    const skillIsA = Math.random() < 0.5;
    const outputA = skillIsA ? outputs.skillOutput : outputs.baselineOutput;
    const outputB = skillIsA ? outputs.baselineOutput : outputs.skillOutput;
    const mcpDeps = detectMcpDependencies(skillContent);
    onProgress?.("scoring", "Scoring comparison...");
    const scores = await scoreComparison(outputA, outputB, prompt, client, mcpDeps.length > 0 ? mcpDeps : undefined);
    // Map scores back to skill/baseline
    const skillContentScore = skillIsA ? scores.contentScoreA : scores.contentScoreB;
    const skillStructureScore = skillIsA ? scores.structureScoreA : scores.structureScoreB;
    const baselineContentScore = skillIsA ? scores.contentScoreB : scores.contentScoreA;
    const baselineStructureScore = skillIsA ? scores.structureScoreB : scores.structureScoreA;
    // Map winner back
    let winner;
    if (scores.winner === "tie") {
        winner = "tie";
    }
    else if (scores.winner === "first") {
        winner = skillIsA ? "skill" : "baseline";
    }
    else {
        winner = skillIsA ? "baseline" : "skill";
    }
    return {
        prompt,
        skillOutput: outputs.skillOutput,
        skillDurationMs: outputs.skillDurationMs,
        skillTokens: outputs.skillTokens,
        baselineOutput: outputs.baselineOutput,
        baselineDurationMs: outputs.baselineDurationMs,
        baselineTokens: outputs.baselineTokens,
        skillContentScore,
        skillStructureScore,
        baselineContentScore,
        baselineStructureScore,
        winner,
    };
}
function clampScore(val) {
    const n = Number(val);
    if (isNaN(n))
        return 3;
    return Math.max(1, Math.min(5, Math.round(n)));
}
//# sourceMappingURL=comparator.js.map