/**
 * AI crawler taxonomy + robots.txt policy.
 *
 * Not all "AI bots" are equal, and blocking them all is usually the wrong call
 * for visibility:
 *
 *   - TRAINING crawlers ingest your content into model weights and never send a
 *     user or a citation back. Blocking these has little downside for traffic
 *     and is what most "block AI" intentions actually mean.
 *   - RETRIEVAL / answer-engine crawlers fetch a page to answer a user's live
 *     query and typically cite + link the source (ChatGPT browsing, Perplexity,
 *     etc.). Blocking these removes you from AI answers — usually undesirable.
 *
 * So we model the two independently. Lists are curated for 2026 and grouped by
 * vendor for maintainability; keep them conservative (only well-attested UA
 * tokens) to avoid accidentally blocking a general search crawler.
 */
/** Crawlers that ingest content for model TRAINING (no citation, no referral). */
export const AI_TRAINING_BOTS = [
    // OpenAI
    'GPTBot',
    // Anthropic
    'ClaudeBot',
    'anthropic-ai',
    // Google (Gemini training; distinct from Googlebot search)
    'Google-Extended',
    // Common Crawl (feeds many training sets)
    'CCBot',
    // ByteDance
    'Bytespider',
    // Apple (Apple Intelligence training; distinct from Applebot search)
    'Applebot-Extended',
    // Meta
    'Meta-ExternalAgent',
    'FacebookBot',
    // Misc training/scraping crawlers
    'Diffbot',
    'Omgilibot',
    'omgili',
    'PanguBot',
    'Timpibot',
    'cohere-ai',
    'ImagesiftBot',
    'PetalBot',
];
/** Answer-engine / live-retrieval crawlers (cite + refer users back). */
export const AI_RETRIEVAL_BOTS = [
    // OpenAI
    'OAI-SearchBot',
    'ChatGPT-User',
    // Perplexity
    'PerplexityBot',
    'Perplexity-User',
    // Anthropic retrieval
    'Claude-Web',
    'Claude-User',
    // Apple search (Siri/Spotlight)
    'Applebot',
    // Amazon (Alexa)
    'Amazonbot',
    // You.com / DuckDuckGo assist
    'YouBot',
    'DuckAssistBot',
];
/**
 * Identify a known AI crawler from a `User-Agent` string. Substring match
 * (case-insensitive) against the curated taxonomy. TRAINING tokens are checked
 * first so the more specific `*-Extended` variants win over their search-crawler
 * namesakes (e.g. `Applebot-Extended` → training, plain `Applebot` → retrieval).
 * Returns `null` for humans and unknown bots — never guesses.
 */
export function detectAiBot(userAgent) {
    if (!userAgent)
        return null;
    const ua = userAgent.toLowerCase();
    for (const token of AI_TRAINING_BOTS) {
        if (ua.includes(token.toLowerCase()))
            return { token, category: 'training' };
    }
    for (const token of AI_RETRIEVAL_BOTS) {
        if (ua.includes(token.toLowerCase()))
            return { token, category: 'retrieval' };
    }
    return null;
}
/**
 * Build the robots.txt User-agent/Disallow line groups for blocked AI bots.
 *
 * Precedence:
 *   1. An explicit `policy` (nuanced training vs retrieval) wins.
 *   2. Otherwise the legacy `blockAIBots` boolean blocks BOTH groups (preserves
 *      pre-nuance behavior for sites that set it).
 *   3. Otherwise nothing is emitted.
 *
 * Returns a flat array of robots.txt lines (each blocked bot gets its own
 * `User-agent` + `Disallow: /` + blank-line group). Pure + deterministic.
 */
export function buildAiBotRobotsLines(policy, legacyBlockAIBots) {
    let blockTraining;
    let blockRetrieval;
    if (policy && (policy.training || policy.retrieval)) {
        blockTraining = policy.training === 'block';
        blockRetrieval = policy.retrieval === 'block';
    }
    else if (legacyBlockAIBots) {
        blockTraining = true;
        blockRetrieval = true;
    }
    else {
        return [];
    }
    const bots = [];
    if (blockTraining)
        bots.push(...AI_TRAINING_BOTS);
    if (blockRetrieval)
        bots.push(...AI_RETRIEVAL_BOTS);
    const lines = [];
    for (const bot of bots) {
        lines.push(`User-agent: ${bot}`);
        lines.push('Disallow: /');
        lines.push('');
    }
    return lines;
}
//# sourceMappingURL=ai-bots.js.map