/**
 * pipeline/probe.ts
 *
 * Tier B probes for new features without evaluation tasks.
 *
 * Phase 4b of the Scenario Matrix implementation.
 *
 * When a content release adds documents for a genuinely new feature (no
 * evaluation tasks exist), a probe provides a directional "usability"
 * signal by analyzing the document content and, optionally, evaluating
 * a generic implementation prompt.
 *
 * Probes are NOT scored evaluations — they answer "are these docs usable?"
 * not "are these docs good enough?" The output is always labeled as
 * directional and never displayed on the same scale as scored evaluations.
 *
 * @see docs/archive/exec-plans/scenario-matrix-implementation/phase-4-content-release-integration.md
 */
// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------
/** Generic probe prompt template */
export const PROBE_PROMPT = `Given the following documentation about a Sanity feature, write a TypeScript
implementation that demonstrates the feature's core functionality.

Use only APIs and patterns described in the documentation. If the documentation
is insufficient to implement something, say so explicitly rather than guessing.

Documentation:
{{docs}}`;
// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------
/**
 * Analyze a model's probe output against the source documentation.
 *
 * Extracts API names from both the documentation and the model output,
 * identifies hallucinations (APIs used but not documented), and classifies
 * the overall usability.
 *
 * This is a pure function — it doesn't call any LLM. The model output
 * is provided as input (from a prior evaluation step or manual run).
 *
 * @param documentContent - The concatenated documentation text
 * @param modelOutput - The model's response to the probe prompt
 * @param documentSlugs - The slugs of the documents that were probed
 * @returns Probe result with usability classification
 */
export function analyzeProbeOutput(documentContent, modelOutput, documentSlugs) {
    const docApis = extractApiNames(documentContent);
    const outputApis = extractApiNames(modelOutput);
    // APIs in the output that aren't in the docs = hallucinations
    const docApiSet = new Set(docApis.map((a) => a.toLowerCase()));
    const hallucinatedApis = outputApis.filter((api) => !docApiSet.has(api.toLowerCase()));
    // APIs in both docs and output
    const apiNamesExtracted = outputApis.filter((api) => docApiSet.has(api.toLowerCase()));
    const producedCode = hasCodeOutput(modelOutput);
    const suggestedTasks = generateTaskSuggestions(documentContent, documentSlugs);
    const usability = classifyUsability(producedCode, apiNamesExtracted.length, hallucinatedApis.length, docApis.length);
    return {
        apiNamesExtracted,
        documentSlugs,
        hallucinatedApis,
        producedCode,
        rawOutput: modelOutput,
        suggestedTasks,
        usability,
    };
}
/**
 * Format a probe result for console output.
 */
export function formatProbeConsole(probe) {
    const lines = [];
    lines.push("🔍 TIER B PROBE RESULT");
    lines.push("");
    lines.push(`  Documents: ${probe.documentSlugs.join(", ")}`);
    lines.push(`  Usability: ${usabilityIcon(probe.usability)} ${probe.usability}`);
    lines.push(`  Produced code: ${probe.producedCode ? "yes" : "no"}`);
    lines.push("");
    if (probe.apiNamesExtracted.length > 0) {
        lines.push("  APIs correctly extracted from docs:");
        for (const api of probe.apiNamesExtracted) {
            lines.push(`    ✅ ${api}`);
        }
        lines.push("");
    }
    if (probe.hallucinatedApis.length > 0) {
        lines.push("  Hallucinated APIs (not in docs):");
        for (const api of probe.hallucinatedApis) {
            lines.push(`    ❌ ${api}`);
        }
        lines.push("");
    }
    if (probe.suggestedTasks.length > 0) {
        lines.push("  Suggested evaluation tasks:");
        for (let i = 0; i < probe.suggestedTasks.length; i++) {
            lines.push(`    ${i + 1}. ${probe.suggestedTasks[i]}`);
        }
        lines.push("");
    }
    return lines.join("\n");
}
// ---------------------------------------------------------------------------
// Formatting
// ---------------------------------------------------------------------------
/**
 * Format a probe result as markdown.
 */
export function formatProbeMarkdown(probe) {
    const lines = [];
    const featureName = inferFeatureName(probe.documentSlugs);
    lines.push(`#### 🔍 New Feature Probe: ${featureName}`);
    lines.push("");
    lines.push("**Status:** No evaluation tasks exist for this feature. " +
        "**Assessment:** Directional only — not a scored evaluation.");
    lines.push("");
    lines.push("**Documents Analyzed:**");
    for (const slug of probe.documentSlugs) {
        lines.push(`- \`${slug}\` (new)`);
    }
    lines.push("");
    lines.push(`**Usability:** ${usabilityIcon(probe.usability)} ${probe.usability}`);
    lines.push("");
    if (probe.apiNamesExtracted.length > 0 || probe.hallucinatedApis.length > 0) {
        lines.push("**Probe Findings:**");
        for (const api of probe.apiNamesExtracted) {
            lines.push(`- ✅ Correctly used \`${api}\``);
        }
        for (const api of probe.hallucinatedApis) {
            lines.push(`- ❌ Hallucinated \`${api}\` (not in docs)`);
        }
        lines.push("");
    }
    if (probe.suggestedTasks.length > 0) {
        lines.push("<details>", "<summary>Suggested evaluation tasks</summary>", "");
        for (let i = 0; i < probe.suggestedTasks.length; i++) {
            lines.push(`${i + 1}. ${probe.suggestedTasks[i]}`);
        }
        lines.push("", "</details>", "");
    }
    return lines.join("\n");
}
/**
 * Generate task scaffolding suggestions from document content analysis.
 *
 * Extracts function names, configuration patterns, and common operations
 * from code blocks in the documentation to suggest evaluation tasks.
 */
export function generateTaskSuggestions(documentContent, documentSlugs) {
    const suggestions = [];
    const featureName = inferFeatureName(documentSlugs);
    // Extract patterns from code blocks
    const codeBlocks = extractCodeBlocks(documentContent);
    const functionNames = extractFunctionNamesFromCode(codeBlocks);
    const configPatterns = detectConfigPatterns(documentContent);
    const crudOperations = detectCrudOperations(documentContent);
    // Generate suggestions based on detected patterns
    if (configPatterns.length > 0) {
        suggestions.push(`Configure ${featureName} in a Sanity Studio project (${configPatterns.join(", ")})`);
    }
    if (functionNames.length > 0) {
        const topFns = functionNames.slice(0, 3).join(", ");
        suggestions.push(`Implement core ${featureName} functionality using ${topFns}`);
    }
    if (crudOperations.length > 0) {
        suggestions.push(`Perform ${crudOperations.join("/")} operations with ${featureName}`);
    }
    // Always suggest an integration task
    if (suggestions.length > 0) {
        suggestions.push(`Integrate ${featureName} with an existing Next.js application`);
    }
    return suggestions;
}
// ---------------------------------------------------------------------------
// Internal helpers
// ---------------------------------------------------------------------------
/** Classify overall usability based on probe metrics */
function classifyUsability(producedCode, correctApis, hallucinatedApis, totalDocApis) {
    if (!producedCode)
        return "not-usable";
    const hallucinationRate = correctApis + hallucinatedApis > 0
        ? hallucinatedApis / (correctApis + hallucinatedApis)
        : 0;
    const coverageRate = totalDocApis > 0 ? correctApis / totalDocApis : 0;
    // Usable: produced code, low hallucination, reasonable API coverage
    if (hallucinationRate < 0.2 && coverageRate > 0.5)
        return "usable";
    // Not usable: high hallucination or very low coverage
    if (hallucinationRate > 0.5 || (coverageRate < 0.2 && totalDocApis > 2)) {
        return "not-usable";
    }
    return "partially-usable";
}
/** Detect configuration patterns in documentation */
function detectConfigPatterns(text) {
    const patterns = [];
    const lower = text.toLowerCase();
    if (/sanity\.config|defineconfig/i.test(lower))
        patterns.push("sanity.config");
    if (/plugin|definePlugin/i.test(lower))
        patterns.push("plugin setup");
    if (/env|environment|api.?key/i.test(lower))
        patterns.push("environment config");
    return patterns;
}
/** Detect CRUD operation patterns in text */
function detectCrudOperations(text) {
    const ops = [];
    const lower = text.toLowerCase();
    if (/\bcreate\b/.test(lower))
        ops.push("create");
    if (/\bread\b|\bfetch\b|\bquery\b|\bget\b/.test(lower))
        ops.push("read");
    if (/\bupdate\b|\bpatch\b|\bmutate\b/.test(lower))
        ops.push("update");
    if (/\bdelete\b|\bremove\b/.test(lower))
        ops.push("delete");
    return ops;
}
/** Extract API-like names from text (function calls, imports, etc.) */
function extractApiNames(text) {
    const apis = new Set();
    // Match function calls: functionName(
    const fnCalls = text.match(/\b([a-zA-Z_]\w{2,})\s*\(/g);
    if (fnCalls) {
        for (const call of fnCalls) {
            const name = call.replace(/\s*\($/, "");
            // Filter out common JS keywords and control flow
            if (!isCommonKeyword(name)) {
                apis.add(name);
            }
        }
    }
    // Match import names: import { name } from
    const imports = text.match(/import\s*\{([^}]+)\}/g);
    if (imports) {
        for (const imp of imports) {
            const names = imp
                .replace(/import\s*\{/, "")
                .replace(/\}/, "")
                .split(",")
                .map((n) => n.trim())
                .filter((n) => n.length > 0 && !n.includes(" as "));
            for (const name of names) {
                apis.add(name);
            }
        }
    }
    return [...apis].sort();
}
/** Extract code blocks from markdown content */
function extractCodeBlocks(text) {
    const blocks = [];
    const regex = /```(?:typescript|ts|javascript|js|tsx|jsx)?\n([\s\S]*?)```/g;
    let match;
    while ((match = regex.exec(text)) !== null) {
        blocks.push(match[1]);
    }
    return blocks;
}
/** Extract function/method names from code blocks */
function extractFunctionNamesFromCode(codeBlocks) {
    const names = new Set();
    for (const block of codeBlocks) {
        // Function declarations
        const fnDecls = block.match(/(?:function|const|let|var)\s+(\w+)/g);
        if (fnDecls) {
            for (const decl of fnDecls) {
                const name = decl.replace(/^(?:function|const|let|var)\s+/, "");
                if (name.length > 2 && !isCommonKeyword(name)) {
                    names.add(name);
                }
            }
        }
        // Method calls: obj.method(
        const methods = block.match(/\.([a-zA-Z_]\w+)\s*\(/g);
        if (methods) {
            for (const method of methods) {
                const name = method.replace(/^\./, "").replace(/\s*\($/, "");
                if (name.length > 2) {
                    names.add(name);
                }
            }
        }
    }
    return [...names].sort();
}
/** Detect whether the model output contains code */
function hasCodeOutput(output) {
    // Check for code fences or obvious code patterns
    if (/```/.test(output))
        return true;
    if (/import\s+\{/.test(output))
        return true;
    if (/(?:const|let|var|function)\s+\w+/.test(output))
        return true;
    return false;
}
/** Infer a human-readable feature name from document slugs */
function inferFeatureName(slugs) {
    if (slugs.length === 0)
        return "Unknown Feature";
    // Find common prefix
    const parts = slugs[0].split("-");
    if (parts.length > 0) {
        // Capitalize and join first 2 parts
        return parts
            .slice(0, 2)
            .map((p) => p.charAt(0).toUpperCase() + p.slice(1))
            .join(" ");
    }
    return slugs[0];
}
/** Check if a name is a common JavaScript keyword */
function isCommonKeyword(name) {
    const keywords = new Set([
        "async",
        "await",
        "break",
        "case",
        "catch",
        "class",
        "const",
        "continue",
        "debugger",
        "default",
        "delete",
        "else",
        "export",
        "extends",
        "finally",
        "for",
        "from",
        "function",
        "get",
        "if",
        "import",
        "in",
        "let",
        "new",
        "of",
        "return",
        "set",
        "switch",
        "throw",
        "try",
        "typeof",
        "var",
        "void",
        "while",
        "with",
        "yield",
        // Common global functions
        "console",
        "log",
        "error",
        "warn",
        "require",
        "module",
        "exports",
        "describe",
        "it",
        "test",
        "expect",
        "assert",
        "JSON",
        "Array",
        "Object",
        "Map",
        "Set",
        "Promise",
        "String",
        "Number",
        "Boolean",
        "Error",
    ]);
    return keywords.has(name);
}
/** Get usability icon */
function usabilityIcon(usability) {
    switch (usability) {
        case "not-usable":
            return "❌";
        case "partially-usable":
            return "🟡";
        case "usable":
            return "✅";
    }
}
