// engine/skill-evaluator.ts — Evaluates skill and rule quality against test scenarios (SPEC-316)
import { readFile } from 'node:fs/promises';
import { existsSync } from 'node:fs';
import { join } from 'node:path';
// ---------------------------------------------------------------------------
// Internal helpers
// ---------------------------------------------------------------------------
/**
 * Read a file from disk, returning empty string if it does not exist.
 */
async function readContent(filePath) {
    if (!existsSync(filePath)) {
        return '';
    }
    return readFile(filePath, 'utf-8');
}
/**
 * Evaluate a single scenario against the given content.
 * A criterion passes if it appears (case-insensitive) somewhere in the content
 * OR in the expectedOutput section of the content.
 */
function evaluateScenario(content, scenario) {
    const lower = content.toLowerCase();
    const expectedLower = scenario.expectedOutput.toLowerCase();
    const failures = [];
    for (const criterion of scenario.criteria) {
        const criterionLower = criterion.toLowerCase();
        const inContent = lower.includes(criterionLower);
        const inExpected = expectedLower.includes(criterionLower);
        if (!inContent && !inExpected) {
            failures.push(criterion);
        }
    }
    const total = scenario.criteria.length;
    const passed = total - failures.length;
    const score = total === 0 ? 1.0 : passed / total;
    return {
        input: scenario.input,
        passed: failures.length === 0,
        score,
        failures,
    };
}
/**
 * Build recommendations based on scenario results.
 */
function buildRecommendations(skillName, results, passRate) {
    const recommendations = [];
    if (passRate < 0.5) {
        recommendations.push(`${skillName} covers fewer than half of tested scenarios — consider a major revision.`);
    }
    else if (passRate < 0.8) {
        recommendations.push(`${skillName} passes ${Math.round(passRate * 100)}% of scenarios — review failing criteria.`);
    }
    // Collect the most common missing criteria across all failures
    const failureCounts = new Map();
    for (const r of results) {
        for (const f of r.failures) {
            failureCounts.set(f, (failureCounts.get(f) ?? 0) + 1);
        }
    }
    const sorted = [...failureCounts.entries()].sort((a, b) => b[1] - a[1]);
    const top = sorted.slice(0, 3);
    for (const [criterion, count] of top) {
        recommendations.push(`Add coverage for "${criterion}" — missing in ${String(count)} scenario(s).`);
    }
    if (recommendations.length === 0) {
        recommendations.push(`${skillName} meets all evaluated criteria — no changes needed.`);
    }
    return recommendations;
}
/**
 * Core evaluation logic used by both evaluateSkill and evaluateRule.
 */
async function evaluate(filePath, displayName, scenarios) {
    const content = await readContent(filePath);
    const scenarioResults = scenarios.map((s) => evaluateScenario(content, s));
    const passedCount = scenarioResults.filter((r) => r.passed).length;
    const passRate = scenarios.length === 0 ? 1.0 : passedCount / scenarios.length;
    const avgScore = scenarioResults.length === 0
        ? 1.0
        : scenarioResults.reduce((sum, r) => sum + r.score, 0) / scenarioResults.length;
    const recommendations = buildRecommendations(displayName, scenarioResults, passRate);
    return {
        skillName: displayName,
        passRate,
        avgScore,
        scenarios: scenarioResults,
        recommendations,
    };
}
// ---------------------------------------------------------------------------
// Path validation
// ---------------------------------------------------------------------------
/**
 * Validates that an identifier (skill or rule name) does not contain
 * path traversal sequences or path separators.
 */
function validateIdentifier(name, field) {
    if (!name ||
        name.includes('/') ||
        name.includes('\\') ||
        name.includes('..') ||
        name.includes('\0')) {
        throw new Error(`Invalid ${field}: "${name}". Must not contain path separators or traversal sequences.`);
    }
}
// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------
/**
 * Evaluate a skill file at the given path against the provided scenarios.
 * Reads .claude/skills/{skillName}.md from the skill path.
 */
export async function evaluateSkill(skillPath, scenarios) {
    const name = skillPath.split('/').pop() ?? skillPath;
    return evaluate(skillPath, name.replace(/\.md$/i, ''), scenarios);
}
/**
 * Evaluate a rule file at the given path against the provided scenarios.
 * Reads .claude/rules/{ruleName}.md from the rule path.
 */
export async function evaluateRule(rulePath, scenarios) {
    const name = rulePath.split('/').pop() ?? rulePath;
    return evaluate(rulePath, name.replace(/\.md$/i, ''), scenarios);
}
/**
 * Resolve the absolute path to a skill file under a project root.
 * Returns the resolved path (may not exist yet — caller decides how to handle).
 */
export function resolveSkillPath(projectPath, skillName) {
    validateIdentifier(skillName, 'skillName');
    const normalized = skillName.endsWith('.md') ? skillName : `${skillName}.md`;
    return join(projectPath, '.claude', 'skills', normalized);
}
/**
 * Resolve the absolute path to a rule file under a project root.
 */
export function resolveRulePath(projectPath, ruleName) {
    validateIdentifier(ruleName, 'ruleName');
    const normalized = ruleName.endsWith('.md') ? ruleName : `${ruleName}.md`;
    return join(projectPath, '.claude', 'rules', normalized);
}
//# sourceMappingURL=skill-evaluator.js.map