/**
 * pipeline/validate.ts
 *
 * Configuration validation for the evaluation pipeline. Checks that all YAML
 * config files are consistent: every task has a canonical mapping, every
 * mapping has a reference solution file, required files exist, etc.
 *
 * All individual validators are exported so they can be tested independently.
 */
import fs from "fs";
import path from "path";
import { tryLoadConfigFile } from "./compiler/config-loader.js";
import { resolveMappings } from "./resolve-mappings.js";
import { FeatureRegistrySchema, formatZodErrors, RubricConfigSchema, ThresholdConfigSchema, } from "./schemas.js";
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
/**
 * Run all validation checks and return a combined result.
 * `valid` is true only if there are zero error-severity issues.
 */
export function validateConfiguration(rootDir) {
    const resolvedDir = path.resolve(rootDir);
    const issues = [
        ...validateModelsYaml(resolvedDir),
        ...validateRubricsYaml(resolvedDir),
        ...validateFeaturesYaml(resolvedDir),
        ...validateThresholdsYaml(resolvedDir),
        ...validateTaskFiles(resolvedDir),
        ...validateReferenceSolutions(resolvedDir),
        ...validateContexts(resolvedDir),
    ];
    const valid = issues.every((issue) => issue.severity !== "error");
    return { issues, valid };
}
/**
 * Check that the canonical-contexts directory exists.
 *
 * Contexts are populated by fetch-docs, which scopes to the tasks
 * actually being evaluated (not every task in the registry). Warning
 * on individual missing files here would fire for every task the user
 * didn't select — pure noise that previously crowded out real errors
 * in the GHA safety-net's tail-of-log capture (W0282). The per-task
 * precondition is enforced by `run-eval-step.ts:checkCanonicalContextsExist`
 * against the filtered task set, where missing files are real errors.
 */
export function validateContexts(rootDir) {
    const source = "validateContexts";
    const issues = [];
    const canonicalDir = path.join(rootDir, "contexts", "canonical");
    if (!fs.existsSync(canonicalDir)) {
        issues.push(warning(source, "contexts/canonical/ directory not found — run 'pnpm fetch-docs' to generate", canonicalDir));
    }
    return issues;
}
/**
 * Check that config/features exists, parses, and conforms to the Zod schema.
 * Also cross-references covered features against actual task files for consistency.
 *
 * Returns warnings (not errors) if the file is missing — the feature registry
 * is optional and doesn't block evaluation.
 */
export function validateFeaturesYaml(rootDir) {
    const source = "validateFeaturesYaml";
    const issues = [];
    const loaded = tryLoadConfigFile("features", rootDir);
    if (!loaded) {
        // Feature registry is optional — warn, don't error
        issues.push(warning(source, "config/features not found — coverage audit unavailable", path.join(rootDir, "config")));
        return issues;
    }
    // Empty features array is valid — it means features come from the preset.
    // The Zod schema requires .min(1) for non-empty configs, but an empty
    // stub is the expected state when the preset provides all features.
    const raw = loaded.data;
    if (Array.isArray(raw?.features) && raw.features.length === 0) {
        return issues;
    }
    const zodResult = FeatureRegistrySchema.safeParse(loaded.data);
    if (!zodResult.success) {
        const lines = formatZodErrors(zodResult.error);
        for (const line of lines) {
            issues.push(error(source, `config/features: ${line.trim()}`, loaded.filePath));
        }
        return issues;
    }
    // Cross-reference: warn if a "covered" feature has no matching task file
    const tasksDir = path.join(rootDir, "tasks");
    if (fs.existsSync(tasksDir)) {
        const taskFiles = new Set(fs
            .readdirSync(tasksDir)
            .filter((f) => /\.(yaml|yml|task\.ts|task\.js)$/.test(f))
            .map((f) => f.replace(/\.(yaml|yml|task\.ts|task\.js)$/, "")));
        for (const feature of zodResult.data.features) {
            if (feature.status === "covered" && feature.area) {
                if (!taskFiles.has(feature.area)) {
                    issues.push(warning(source, `Feature '${feature.id}' is status: covered with area: '${feature.area}', but tasks/${feature.area} does not exist`, loaded.filePath));
                }
            }
            // Warn if an "uncovered" feature has a matching task file
            if (feature.status === "uncovered" && feature.area) {
                if (taskFiles.has(feature.area)) {
                    issues.push(warning(source, `Feature '${feature.id}' is status: uncovered but tasks/${feature.area} task file exists — consider updating status to 'covered'`, loaded.filePath));
                }
            }
        }
        // Check for duplicate feature IDs
        const ids = new Set();
        for (const feature of zodResult.data.features) {
            if (ids.has(feature.id)) {
                issues.push(error(source, `Duplicate feature id '${feature.id}' in config/features`, loaded.filePath));
            }
            ids.add(feature.id);
        }
    }
    return issues;
}
/**
 * Check that config/models exists, parses, has at least one model with an id
 * and label, and has a grader defined.
 */
export function validateModelsYaml(rootDir) {
    const source = "validateModelsYaml";
    const issues = [];
    let data;
    try {
        data = loadConfigFileForValidation("models", rootDir);
    }
    catch {
        issues.push(error(source, "config/models not found", path.join(rootDir, "config")));
        return issues;
    }
    if (!data || typeof data !== "object") {
        issues.push(error(source, "config/models did not parse to an object", path.join(rootDir, "config")));
        return issues;
    }
    // Check models array
    if (!Array.isArray(data.models)) {
        issues.push(error(source, "config/models is missing a 'models' array", path.join(rootDir, "config")));
    }
    else if (data.models.length === 0) {
        issues.push(error(source, "config/models has an empty 'models' array", path.join(rootDir, "config")));
    }
    else {
        for (const [i, model] of data.models.entries()) {
            if (!model.id) {
                issues.push(error(source, `models[${i}] is missing 'id'`, path.join(rootDir, "config")));
            }
            if (!model.label) {
                issues.push(error(source, `models[${i}] is missing 'label'`, path.join(rootDir, "config")));
            }
            if (model.timeoutMs !== undefined &&
                (typeof model.timeoutMs !== "number" ||
                    !Number.isInteger(model.timeoutMs) ||
                    model.timeoutMs <= 0)) {
                issues.push(error(source, `models[${i}] has invalid 'timeoutMs' (must be a positive integer)`, path.join(rootDir, "config")));
            }
        }
    }
    // Check evalBudgetMs
    if (data.evalBudgetMs !== undefined &&
        (typeof data.evalBudgetMs !== "number" ||
            !Number.isInteger(data.evalBudgetMs) ||
            data.evalBudgetMs <= 0)) {
        issues.push(error(source, "config/models has invalid 'evalBudgetMs' (must be a positive integer)", path.join(rootDir, "config")));
    }
    // Check grader
    if (!data.grader) {
        issues.push(error(source, "config/models is missing a 'grader' section", path.join(rootDir, "config")));
    }
    else if (!data.grader.id) {
        issues.push(error(source, "config/models grader is missing 'id'", path.join(rootDir, "config")));
    }
    return issues;
}
/** Helper: load a config file for validation (throws on not found) */
function loadConfigFileForValidation(name, rootDir) {
    const result = tryLoadConfigFile(name, rootDir);
    if (!result)
        throw new Error(`Config "${name}" not found`);
    return result.data;
}
// ---------------------------------------------------------------------------
// Validators
// ---------------------------------------------------------------------------
/**
 * Check that reference solution files exist on disk for every task
 * that declares a reference_solution path. Reads from inline task
 * definitions via resolveMappings.
 *
 * NOTE: When tasks come from the Content Lake, reference solutions also
 * live there (as ailf.referenceSolution documents). This validator only
 * applies to YAML-based tasks with local file paths.
 */
export function validateReferenceSolutions(rootDir) {
    const source = "validateReferenceSolutions";
    const issues = [];
    const canonicalDir = path.join(rootDir, "canonical");
    const mappings = resolveMappings(rootDir);
    for (const [area, areaConfig] of Object.entries(mappings.feature_areas)) {
        if (!areaConfig?.tasks)
            continue;
        for (const task of areaConfig.tasks) {
            const taskLabel = `${area}/${task.id}`;
            if (!task.reference_solution) {
                issues.push(error(source, `${taskLabel} is missing 'reference_solution'`));
                continue;
            }
            const solutionPath = path.join(canonicalDir, task.reference_solution);
            if (!fs.existsSync(solutionPath)) {
                issues.push(error(source, `${taskLabel} reference_solution not found: ${task.reference_solution}`, solutionPath));
            }
        }
    }
    return issues;
}
/**
 * Check that config/rubrics exists, parses, and conforms to the Zod schema.
 * Returns the set of valid template keys for cross-referencing by task
 * validation.
 */
export function validateRubricsYaml(rootDir) {
    const source = "validateRubricsYaml";
    const issues = [];
    const loaded = tryLoadConfigFile("rubrics", rootDir);
    if (!loaded) {
        issues.push(error(source, "config/rubrics not found", path.join(rootDir, "config")));
        return issues;
    }
    const zodResult = RubricConfigSchema.safeParse(loaded.data);
    if (!zodResult.success) {
        const lines = formatZodErrors(zodResult.error);
        for (const line of lines) {
            issues.push(error(source, `config/rubrics: ${line.trim()}`, loaded.filePath));
        }
    }
    return issues;
}
/**
 * Check that task definition files exist.
 *
 * Tasks live as `*.task.ts` files in mode subdirectories (e.g.
 * `tasks/literacy/groq.task.ts`). Legacy YAML task files are no longer
 * used. Warns only if no task files are found at all.
 */
export function validateTaskFiles(rootDir) {
    const source = "validateTaskFiles";
    const issues = [];
    const tasksDir = path.join(rootDir, "tasks");
    if (!fs.existsSync(tasksDir)) {
        issues.push(warning(source, "tasks/ directory not found (using Content Lake tasks?)", tasksDir));
        return issues;
    }
    const taskAreas = collectTaskAreas(tasksDir);
    if (taskAreas.size === 0) {
        issues.push(warning(source, "No task files found in tasks/ (using Content Lake tasks?)", tasksDir));
    }
    return issues;
}
/**
 * Check that config/thresholds exists, parses, and conforms to the Zod schema.
 *
 * Returns warnings (not errors) if the file is missing — thresholds are
 * optional and don't block evaluation. They only activate when
 * `ailf report readiness` or severity-aware sink routing is used.
 */
export function validateThresholdsYaml(rootDir) {
    const source = "validateThresholdsYaml";
    const issues = [];
    const loaded = tryLoadConfigFile("thresholds", rootDir);
    if (!loaded) {
        // Thresholds are optional — warn, don't error
        issues.push(warning(source, "config/thresholds not found — readiness gates and threshold alerts unavailable", path.join(rootDir, "config")));
        return issues;
    }
    const zodResult = ThresholdConfigSchema.safeParse(loaded.data);
    if (!zodResult.success) {
        const lines = formatZodErrors(zodResult.error);
        for (const line of lines) {
            issues.push(error(source, `config/thresholds: ${line.trim()}`, loaded.filePath));
        }
        return issues;
    }
    // Cross-reference: warn if an area override references an area with no task file
    if (zodResult.data.areas) {
        const tasksDir = path.join(rootDir, "tasks");
        const taskAreas = collectTaskAreas(tasksDir);
        for (const areaName of Object.keys(zodResult.data.areas)) {
            if (!taskAreas.has(areaName)) {
                issues.push(warning(source, `config/thresholds: area override '${areaName}' has no matching task file`, loaded.filePath));
            }
        }
    }
    return issues;
}
function error(source, message, filePath) {
    return {
        message,
        severity: "error",
        source,
        ...(filePath && { path: filePath }),
    };
}
/**
 * Collect task area names from all subdirectories of `tasksDir`.
 *
 * Task files live in mode subdirectories (e.g. `tasks/literacy/groq.task.ts`).
 * Returns a set of basenames without the `.task.ts`/`.task.js` extension.
 */
function collectTaskAreas(tasksDir) {
    if (!fs.existsSync(tasksDir))
        return new Set();
    const areas = new Set();
    const taskFilePattern = /\.task\.(ts|js)$/;
    for (const entry of fs.readdirSync(tasksDir, { withFileTypes: true })) {
        if (entry.isDirectory()) {
            const subdir = path.join(tasksDir, entry.name);
            for (const file of fs.readdirSync(subdir)) {
                if (taskFilePattern.test(file)) {
                    areas.add(file.replace(taskFilePattern, ""));
                }
            }
        }
        // Also check top-level task files for backwards compatibility
        if (entry.isFile() && taskFilePattern.test(entry.name)) {
            areas.add(entry.name.replace(taskFilePattern, ""));
        }
    }
    return areas;
}
// ---------------------------------------------------------------------------
// Main entry point
// ---------------------------------------------------------------------------
function warning(source, message, filePath) {
    return {
        message,
        severity: "warning",
        source,
        ...(filePath && { path: filePath }),
    };
}
