/**
 * workflow/core/judgeScorer.ts
 * Judge-based scoring system for ensemble response evaluation
 */
import { AIProviderFactory } from "../../core/factory.js";
import { logger } from "../../utils/logger.js";
import { SpanSerializer, SpanType, SpanStatus, getMetricsAggregator, } from "../../observability/index.js";
import { MAX_REASONING_LENGTH } from "../config.js";
import { WorkflowError } from "../types.js";
const functionTag = "JudgeScorer";
// ============================================================================
// SCORING FUNCTIONS
// ============================================================================
/**
 * Execute judge scoring on ensemble responses
 * @param options - Scoring options including judges and responses
 * @returns Score result with judge evaluation
 */
export async function scoreEnsemble(options) {
    const startTime = Date.now();
    const { judges, responses, originalPrompt, systemPrompt, timeout, workflowDefaults, } = options;
    const span = SpanSerializer.createSpan(SpanType.WORKFLOW, "workflow.judge", {
        "workflow.operation": "judge",
        "workflow.judge_count": judges.length,
        "workflow.response_count": responses.length,
        "workflow.pattern": judges.length > 1 ? "multi-judge" : "single-judge",
    });
    logger.info(`[${functionTag}] Starting judge scoring`, {
        judgeCount: judges.length,
        responseCount: responses.length,
    });
    try {
        // Filter successful responses for evaluation
        const successfulResponses = responses.filter((r) => r.status === "success" && r.content.trim() !== "");
        if (successfulResponses.length === 0) {
            throw new WorkflowError("No successful responses to evaluate", {
                code: "NO_RESPONSES_TO_EVALUATE",
                workflowId: "judge",
                phase: "judge",
                retryable: false,
            });
        }
        if (judges.length === 1) {
            // Single judge scoring
            const judgeResult = await executeSingleJudge(judges[0], successfulResponses, originalPrompt, systemPrompt, timeout, workflowDefaults?.judgePrompt);
            const judgeTime = Date.now() - startTime;
            span.durationMs = judgeTime;
            const endedSpan = SpanSerializer.endSpan(span, SpanStatus.OK);
            getMetricsAggregator().recordSpan(endedSpan);
            return {
                scores: judgeResult,
                judgeTime,
            };
        }
        else {
            // Multi-judge voting
            const multiJudgeResult = await executeMultiJudge(judges, successfulResponses, originalPrompt, systemPrompt, timeout, workflowDefaults?.judgePrompt);
            const judgeTime = Date.now() - startTime;
            span.durationMs = judgeTime;
            const endedSpan = SpanSerializer.endSpan(span, SpanStatus.OK);
            getMetricsAggregator().recordSpan(endedSpan);
            return {
                scores: multiJudgeResult,
                judgeTime,
            };
        }
    }
    catch (error) {
        const err = error;
        logger.error(`[${functionTag}] Judge scoring failed`, {
            error: err.message,
        });
        span.durationMs = Date.now() - startTime;
        const endedSpan = SpanSerializer.endSpan(span, SpanStatus.ERROR, err.message);
        getMetricsAggregator().recordSpan(endedSpan);
        const workflowError = error instanceof WorkflowError
            ? error
            : new WorkflowError(err.message, {
                code: "JUDGE_SCORING_ERROR",
                workflowId: "judge",
                phase: "judge",
                retryable: true,
            });
        return {
            scores: createEmptyScores(judges[0], responses),
            judgeTime: Date.now() - startTime,
            error: workflowError,
        };
    }
}
/**
 * Execute single judge evaluation
 * @param judge - Judge configuration
 * @param responses - Successful ensemble responses
 * @param originalPrompt - Original user prompt
 * @param systemPrompt - Optional system prompt
 * @param timeout - Judge timeout in milliseconds
 * @returns Judge scores with evaluation
 */
async function executeSingleJudge(judge, responses, originalPrompt, systemPrompt, timeout, workflowDefaultJudgePrompt) {
    const startTime = Date.now();
    logger.debug(`[${functionTag}] Executing single judge`, {
        provider: judge.provider,
        model: judge.model,
    });
    // Resolve judge prompt with hierarchical fallback:
    // 1. Judge-specific customPrompt (highest priority)
    // 2. Workflow-level default judge prompt
    // 3. Built-in default template
    const resolvedJudgePrompt = judge.customPrompt || workflowDefaultJudgePrompt;
    // Create judge prompt (will use resolvedJudgePrompt if provided, otherwise default template)
    const judgePrompt = createJudgePrompt(judge, responses, originalPrompt, resolvedJudgePrompt);
    // Execute judge
    const provider = await AIProviderFactory.createProvider(judge.provider, judge.model);
    const result = await provider.generate({
        prompt: judgePrompt,
        systemPrompt: systemPrompt || judge.systemPrompt,
        temperature: judge.temperature || 0.1,
        maxTokens: judge.maxTokens || 2000,
        timeout: timeout || judge.timeout || 10000,
    });
    // Parse judge response
    const parsed = parseJudgeResponse(result?.content || "", responses, judge);
    // Build JudgeScores
    const judgeScores = {
        judgeProvider: judge.provider,
        judgeModel: judge.model,
        scores: parsed.scores,
        ranking: parsed.ranking,
        bestResponse: parsed.bestResponse,
        criteria: judge.criteria,
        reasoning: parsed.reasoning,
        synthesizedResponse: parsed.synthesizedResponse, // Include synthesized response if present
        confidenceInJudgment: parsed.confidenceInJudgment,
        judgeTime: Date.now() - startTime,
        timestamp: new Date().toISOString(),
    };
    logger.debug(`[${functionTag}] Single judge completed`, {
        bestResponse: judgeScores.bestResponse,
        hasSynthesizedResponse: !!parsed.synthesizedResponse,
        judgeTime: judgeScores.judgeTime,
    });
    return judgeScores;
}
/**
 * Execute multi-judge voting
 * @param judges - Array of judge configurations
 * @param responses - Successful ensemble responses
 * @param originalPrompt - Original user prompt
 * @param systemPrompt - Optional system prompt
 * @param timeout - Judge timeout in milliseconds
 * @returns Multi-judge scores with aggregated results
 */
async function executeMultiJudge(judges, responses, originalPrompt, systemPrompt, timeout, workflowDefaultJudgePrompt) {
    const startTime = Date.now();
    logger.debug(`[${functionTag}] Executing multi-judge voting`, {
        judgeCount: judges.length,
    });
    // Execute all judges in parallel
    const judgePromises = judges.map((judge) => executeSingleJudge(judge, responses, originalPrompt, systemPrompt, timeout, workflowDefaultJudgePrompt).catch((error) => {
        logger.warn(`[${functionTag}] Judge failed`, {
            provider: judge.provider,
            model: judge.model,
            error: error.message,
        });
        return createEmptyJudgeScores(judge, responses);
    }));
    const judgeResults = await Promise.all(judgePromises);
    // Aggregate scores using average (can be configurable in future)
    const aggregated = aggregateJudgeScores(judgeResults, "average");
    const multiJudgeScores = {
        judges: judgeResults,
        averageScores: aggregated.averageScores,
        aggregatedRanking: aggregated.ranking,
        consensusLevel: calculateConsensusLevel(judgeResults),
        bestResponse: aggregated.bestResponse,
        confidence: aggregated.confidence,
        votingStrategy: "average",
        // Expose unified interface fields
        judgeProvider: judgeResults[0]?.judgeProvider,
        judgeModel: `multi-judge-${judges.length}`,
        scores: aggregated.averageScores,
        ranking: aggregated.ranking,
        reasoning: aggregated.reasoning,
        confidenceInJudgment: aggregated.confidence,
        criteria: judges[0]?.criteria || [],
        judgeTime: Date.now() - startTime,
        timestamp: new Date().toISOString(),
    };
    logger.debug(`[${functionTag}] Multi-judge completed`, {
        bestResponse: multiJudgeScores.bestResponse,
        consensusLevel: multiJudgeScores.consensusLevel,
        judgeTime: multiJudgeScores.judgeTime,
    });
    return multiJudgeScores;
}
// ============================================================================
// HELPER FUNCTIONS
// ============================================================================
/**
 * Create judge evaluation prompt
 * @param judge - Judge configuration
 * @param responses - Ensemble responses to evaluate
 * @param originalPrompt - Original user prompt
 * @param customPrompt - Custom evaluation prompt (overrides default)
 * @returns Formatted judge prompt
 */
function createJudgePrompt(judge, responses, originalPrompt, customPrompt) {
    // If custom prompt provided, use it
    if (customPrompt) {
        logger.debug(`[${functionTag}] Using custom judge prompt`);
        return customPrompt;
    }
    // Build response blocks
    const responseBlocks = responses
        .map((r, index) => {
        const identifier = `response-${index}`;
        const modelInfo = judge.blindEvaluation
            ? `Response ${index + 1}`
            : `${r.provider}/${r.model}`;
        return `
<response id="${identifier}">
<model>${modelInfo}</model>
<content>
${r.content}
</content>
</response>`;
    })
        .join("\n");
    const criteriaList = judge.criteria
        .map((c, i) => `${i + 1}. ${c}`)
        .join("\n");
    // If synthesis is enabled, judge creates improved response
    if (judge.synthesizeImprovedResponse) {
        return `You are an expert AI evaluator and synthesizer. Your task is to:
1. Evaluate all responses
2. Synthesize an IMPROVED final response that combines their strengths

USER QUESTION:
${originalPrompt}

RESPONSES TO EVALUATE:
${responseBlocks}

EVALUATION CRITERIA:
${criteriaList}

INSTRUCTIONS:
1. Score each response on a scale of 0-100 (0 = poor, 100 = excellent)
2. Consider all evaluation criteria listed above
3. Provide a ranking of responses from best to worst
4. Identify the single best response
5. Provide brief reasoning for your evaluation (max 200 characters)
6. **SYNTHESIZE an improved response** that:
   - Combines the best elements from all responses
   - Addresses any weaknesses identified in the evaluation
   - Maintains accuracy and technical correctness
   - Is more complete and higher quality than any single response
   - Directly answers the user's question (no meta-commentary)
7. Rate your confidence in this judgment (0.0 to 1.0)

Respond in JSON format:
{
  "scores": {
    "response-0": 85,
    "response-1": 92
  },
  "ranking": ["response-1", "response-0"],
  "bestResponse": "response-1",
  "reasoning": "Brief explanation of evaluation",
  "synthesizedResponse": "Your improved, synthesized response here",
  "confidenceInJudgment": 0.9
}`;
    }
    // Standard evaluation (no synthesis)
    return `You are an expert AI evaluator. Evaluate the following responses to the user's question.

USER QUESTION:
${originalPrompt}

RESPONSES TO EVALUATE:
${responseBlocks}

EVALUATION CRITERIA:
${criteriaList}

INSTRUCTIONS:
1. Score each response on a scale of 0-100 (0 = poor, 100 = excellent)
2. Consider all evaluation criteria listed above
3. Provide a ranking of responses from best to worst
4. Identify the single best response
5. Provide brief reasoning for your evaluation (max 200 characters)
6. Rate your confidence in this judgment (0.0 to 1.0)

Respond in JSON format:
{
  "scores": {
    "response-0": 85,
    "response-1": 92
  },
  "ranking": ["response-1", "response-0"],
  "bestResponse": "response-1",
  "reasoning": "Brief explanation of evaluation",
  "confidenceInJudgment": 0.9
}`;
}
/**
 * Parse judge response to extract scores
 * @param content - Raw judge response content
 * @param responses - Original ensemble responses
 * @param _judge - Judge configuration (unused)
 * @returns Parsed judge response with scores
 */
function parseJudgeResponse(content, responses, _judge) {
    try {
        // Try to extract JSON from response
        const jsonMatch = content.match(/\{[\s\S]*\}/);
        if (!jsonMatch) {
            logger.warn(`[${functionTag}] No JSON found in judge response`);
            return createFallbackScores(responses);
        }
        const parsed = JSON.parse(jsonMatch[0]);
        // Validate and normalize scores to 0-100 range
        const scores = {};
        Object.keys(parsed.scores || {}).forEach((key) => {
            const score = Number(parsed.scores[key]);
            scores[key] = Math.max(0, Math.min(100, score));
        });
        // Ensure all responses have scores
        responses.forEach((_, index) => {
            const key = `response-${index}`;
            if (!(key in scores)) {
                scores[key] = 50; // Default neutral score
            }
        });
        return {
            scores,
            ranking: parsed.ranking || generateRankingFromScores(scores),
            bestResponse: parsed.bestResponse || findBestResponse(scores),
            reasoning: truncateReasoning(parsed.reasoning || "No reasoning provided"),
            synthesizedResponse: parsed.synthesizedResponse, // Extract synthesized response if present
            confidenceInJudgment: normalizeConfidence(parsed.confidenceInJudgment),
        };
    }
    catch (error) {
        logger.warn(`[${functionTag}] Failed to parse judge response`, {
            error: error.message,
        });
        return createFallbackScores(responses);
    }
}
/**
 * Create fallback scores when parsing fails
 * @param responses - Ensemble responses
 * @returns Default scores with equal values
 */
function createFallbackScores(responses) {
    const scores = {};
    const ranking = [];
    responses.forEach((_, index) => {
        const key = `response-${index}`;
        scores[key] = 50; // Neutral score
        ranking.push(key);
    });
    return {
        scores,
        ranking,
        bestResponse: ranking[0],
        reasoning: "Unable to parse judge evaluation",
        confidenceInJudgment: 0.5,
    };
}
/**
 * Generate ranking from scores
 * @param scores - Score record
 * @returns Array of response IDs sorted by score descending
 */
function generateRankingFromScores(scores) {
    return Object.keys(scores).sort((a, b) => scores[b] - scores[a]);
}
/**
 * Find best response from scores
 * @param scores - Score record
 * @returns Response ID with highest score
 */
function findBestResponse(scores) {
    let bestId = "";
    let bestScore = -1;
    Object.keys(scores).forEach((key) => {
        if (scores[key] > bestScore) {
            bestScore = scores[key];
            bestId = key;
        }
    });
    return bestId || Object.keys(scores)[0];
}
/**
 * Truncate reasoning to max 200 characters
 * @param reasoning - Reasoning text
 * @returns Truncated reasoning
 */
function truncateReasoning(reasoning) {
    if (reasoning.length <= MAX_REASONING_LENGTH) {
        return reasoning;
    }
    return reasoning.substring(0, MAX_REASONING_LENGTH - 3) + "...";
}
/**
 * Normalize confidence to 0-1 range
 * @param confidence - Confidence value
 * @returns Normalized confidence between 0 and 1
 */
function normalizeConfidence(confidence) {
    if (typeof confidence !== "number") {
        return 0.5;
    }
    return Math.max(0, Math.min(1, confidence));
}
/**
 * Aggregate multiple judge scores
 * @param judgeResults - Array of judge score results
 * @param _strategy - Aggregation strategy (currently only 'average')
 * @returns Aggregated scores and ranking
 */
function aggregateJudgeScores(judgeResults, _strategy) {
    // Collect all response IDs
    const responseIds = new Set();
    judgeResults.forEach((result) => {
        Object.keys(result.scores).forEach((id) => responseIds.add(id));
    });
    // Calculate average scores
    const averageScores = {};
    responseIds.forEach((id) => {
        const scores = judgeResults
            .map((result) => result.scores[id])
            .filter((score) => score !== undefined);
        if (scores.length > 0) {
            averageScores[id] =
                scores.reduce((sum, score) => sum + score, 0) / scores.length;
        }
        else {
            averageScores[id] = 50; // Default
        }
    });
    // Generate ranking from average scores
    const ranking = generateRankingFromScores(averageScores);
    const bestResponse = ranking[0];
    // Calculate aggregate confidence
    const confidences = judgeResults
        .map((r) => r.confidenceInJudgment || 0.5)
        .filter((c) => c > 0);
    const confidence = confidences.length > 0
        ? confidences.reduce((sum, c) => sum + c, 0) / confidences.length
        : 0.5;
    // Aggregate reasoning
    const reasoning = `Aggregated from ${judgeResults.length} judges`;
    return {
        averageScores,
        ranking,
        bestResponse,
        confidence,
        reasoning,
    };
}
/**
 * Calculate consensus level between judges
 * @param judgeResults - Array of judge score results
 * @returns Consensus level between 0 and 1
 */
function calculateConsensusLevel(judgeResults) {
    if (judgeResults.length < 2) {
        return 1.0; // Perfect consensus with single judge
    }
    // Calculate agreement on best response
    const bestResponses = judgeResults.map((r) => r.bestResponse);
    const modeCounts = new Map();
    bestResponses.forEach((response) => {
        if (response) {
            modeCounts.set(response, (modeCounts.get(response) || 0) + 1);
        }
    });
    const maxCount = Math.max(...Array.from(modeCounts.values()));
    return maxCount / judgeResults.length;
}
/**
 * Create empty judge scores for error cases
 * @param judge - Judge configuration
 * @param responses - Ensemble responses
 * @returns Empty judge scores
 */
function createEmptyJudgeScores(judge, responses) {
    const scores = {};
    responses.forEach((_, index) => {
        scores[`response-${index}`] = 50;
    });
    return {
        judgeProvider: judge.provider,
        judgeModel: judge.model,
        scores,
        criteria: judge.criteria,
        judgeTime: 0,
        timestamp: new Date().toISOString(),
    };
}
/**
 * Create empty scores for error cases
 * @param judge - Judge configuration
 * @param responses - Ensemble responses
 * @returns Empty judge scores
 */
function createEmptyScores(judge, responses) {
    return createEmptyJudgeScores(judge, responses);
}
/**
 * Get best response from judge scores
 * @param scores - Judge scores or multi-judge scores
 * @param responses - Original ensemble responses
 * @returns Best ensemble response
 */
export function getBestResponse(scores, responses) {
    const bestId = scores.bestResponse;
    if (!bestId) {
        return undefined;
    }
    const index = parseInt(bestId.replace("response-", ""), 10);
    return responses[index];
}
/**
 * Get ranked responses
 * @param scores - Judge scores or multi-judge scores
 * @param responses - Original ensemble responses
 * @returns Responses sorted by ranking
 */
export function getRankedResponses(scores, responses) {
    if (!scores.ranking || scores.ranking.length === 0) {
        return responses;
    }
    return scores.ranking
        .map((id) => {
        const index = parseInt(id.replace("response-", ""), 10);
        return responses[index];
    })
        .filter((r) => r !== undefined);
}
//# sourceMappingURL=judgeScorer.js.map