/**
 * Web Fetch Tool
 *
 * Fetches content from a URL, converts HTML to markdown,
 * and processes it with an LLM based on a user prompt.
 */
import { generateObject } from "ai";
import TurndownService from "turndown";
import { z } from "zod";
import { getLogger, getErrorMeta } from "../../../../util/logger.js";
import { createToolFactory, PermissionLevel, ToolCategory } from "../types.js";
/**
 * Schema for structured LLM response.
 * Uses generateObject pattern from safety-classifier for Clark API compatibility.
 */
const WebFetchResponseSchema = z.object({
    content: z
        .string()
        .describe("The extracted information as plain markdown text (NOT JSON). Use markdown formatting for headers, lists, bold, etc."),
});
/**
 * System prompt for web content extraction.
 */
const WEB_FETCH_SYSTEM_PROMPT = `You are a helpful assistant that extracts and summarizes information from web page content.
Analyze the provided web page content and respond to the user's prompt with relevant information.
Be concise and focus on the information requested.

IMPORTANT: Return your response as plain markdown text, NOT as JSON. Use markdown formatting (headers, lists, bold, etc.) to structure your response clearly.`;
/**
 * Cache duration in milliseconds (15 minutes).
 */
const CACHE_DURATION_MS = 15 * 60 * 1000;
/**
 * Maximum content length to send to LLM (characters).
 */
const MAX_CONTENT_LENGTH = 100_000;
/**
 * Simple TTL-based cache for fetched content.
 */
class WebFetchCache {
    cache = new Map();
    get(url) {
        const entry = this.cache.get(url);
        if (!entry)
            return undefined;
        if (Date.now() - entry.timestamp > CACHE_DURATION_MS) {
            this.cache.delete(url);
            return undefined;
        }
        return entry.content;
    }
    set(url, content) {
        this.cache.set(url, { content, timestamp: Date.now() });
    }
    clear() {
        this.cache.clear();
    }
}
// Module-level cache instance
const fetchCache = new WebFetchCache();
/**
 * Turndown service for HTML to markdown conversion.
 */
const turndownService = new TurndownService({
    headingStyle: "atx",
    codeBlockStyle: "fenced",
});
// Remove script, style, and other non-content elements
turndownService.remove(["script", "style", "nav", "footer", "header", "aside"]);
/**
 * Converts HTML to markdown.
 *
 * @param html - HTML content
 * @returns Markdown content
 */
function htmlToMarkdown(html) {
    return turndownService.turndown(html);
}
/**
 * Upgrades HTTP URLs to HTTPS.
 *
 * @param url - URL to upgrade
 * @returns HTTPS URL
 */
function upgradeToHttps(url) {
    if (url.startsWith("http://")) {
        return url.replace("http://", "https://");
    }
    return url;
}
/**
 * Validates a URL is safe to fetch (SSRF protection).
 * Blocks requests to private IP ranges and localhost.
 *
 * @param url - URL to validate
 * @returns true if the URL is safe to fetch
 */
function isAllowedUrl(url) {
    const parsed = new URL(url);
    const hostname = parsed.hostname.toLowerCase();
    // Block localhost
    if (hostname === "localhost") {
        return false;
    }
    // Block private IP ranges
    if (hostname.startsWith("127.") || // Loopback
        hostname.startsWith("10.") || // Class A private
        hostname.startsWith("192.168.") || // Class C private
        hostname.startsWith("169.254.") || // Link-local
        hostname.startsWith("172.") // Check Class B private range (172.16-31.x.x)
    ) {
        // For 172.x, need to check if it's in the 172.16.0.0 - 172.31.255.255 range
        if (hostname.startsWith("172.")) {
            const parts = hostname.split(".");
            const secondOctet = parseInt(parts[1] ?? "0", 10);
            if (secondOctet >= 16 && secondOctet <= 31) {
                return false;
            }
        }
        else {
            return false;
        }
    }
    // Block IPv6 localhost and private ranges
    if (hostname === "[::1]" ||
        hostname.startsWith("[fc") ||
        hostname.startsWith("[fd") ||
        hostname.startsWith("[fe80")) {
        return false;
    }
    return true;
}
/**
 * Truncates content if it exceeds the maximum length.
 *
 * @param content - Content to truncate
 * @param maxLength - Maximum length
 * @returns Truncated content with note if applicable
 */
function truncateContent(content, maxLength) {
    if (content.length <= maxLength) {
        return content;
    }
    return (content.slice(0, maxLength) +
        "\n\n[Content truncated - original length: " +
        content.length +
        " characters]");
}
/**
 * Web fetch tool factory.
 *
 * Fetches URL content, converts to markdown, and processes with LLM.
 */
export const webFetchToolFactory = createToolFactory("web_fetch", ({ llmProvider }) => ({
    description: `
Fetches content from a URL and extracts specific information using an AI model.

WHEN TO USE:
- After web_search returns relevant URLs that need deeper exploration
- When the user provides a documentation URL, GitHub link, or any webpage
- To get detailed information from official docs, release notes, or changelogs
- To verify information found in search snippets by reading the full page
- After installing a new library - read the official docs to learn correct usage patterns

Use this tool to follow up on web_search results when snippets don't provide enough detail.

Usage:
- Provide a URL and a prompt describing what information to extract
- HTTP URLs are automatically upgraded to HTTPS
- HTML content is converted to markdown before processing
- Results are cached for 15 minutes for faster repeated access

When a URL redirects to a different host, the tool will inform you
and provide the redirect URL. Make a new request with that URL.

Parameters:
- url: The URL to fetch content from (required)
- prompt: What information to extract from the page (required)

Examples:
- Fetch API docs: url="https://docs.example.com/api", prompt="List all available endpoints"
- Get pricing info: url="https://example.com/pricing", prompt="What are the pricing tiers?"
- Extract changelog: url="https://github.com/org/repo/releases", prompt="Summarize recent changes"
`,
    category: ToolCategory.GENERAL,
    defaultPermission: PermissionLevel.ALLOW,
    readOnly: true,
    inputSchema: z.object({
        url: z.string().url().describe("The URL to fetch content from"),
        prompt: z
            .string()
            .describe("The prompt describing what information to extract"),
    }),
    execute: async ({ url, prompt }, options) => {
        const logger = getLogger();
        try {
            // Upgrade HTTP to HTTPS
            const secureUrl = upgradeToHttps(url);
            // SSRF protection: block private IPs and localhost
            if (!isAllowedUrl(secureUrl)) {
                throw new Error("URL not allowed: cannot fetch from localhost or private IP ranges");
            }
            // Check cache first
            let markdown = fetchCache.get(secureUrl);
            if (!markdown) {
                // Fetch the URL
                const response = await fetch(secureUrl, {
                    signal: options?.abortSignal,
                    redirect: "follow",
                    headers: {
                        "User-Agent": "Mozilla/5.0 (compatible; WebFetch/1.0; +https://superblocks.com)",
                        Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
                    },
                });
                if (!response.ok) {
                    throw new Error(`Fetch returned status ${response.status}`);
                }
                // Check for cross-host redirect
                const originalHost = new URL(secureUrl).host;
                const finalHost = new URL(response.url).host;
                if (response.redirected && originalHost !== finalHost) {
                    return {
                        type: "redirect",
                        redirectUrl: response.url,
                        message: `URL redirected to a different host. Make a new request with the redirect URL: ${response.url}`,
                    };
                }
                // Get content and convert to markdown
                const html = await response.text();
                markdown = htmlToMarkdown(html);
                // Cache the result
                fetchCache.set(secureUrl, markdown);
            }
            // Truncate content if needed
            const truncatedContent = truncateContent(markdown, MAX_CONTENT_LENGTH);
            // Process with LLM using generateObject pattern (like safety-classifier)
            const model = llmProvider.modelForTask("fast");
            const { object: result } = await generateObject({
                model,
                schema: WebFetchResponseSchema,
                system: WEB_FETCH_SYSTEM_PROMPT,
                prompt: `User prompt: ${prompt}\n\nWeb page content:\n${truncatedContent}`,
                abortSignal: options?.abortSignal,
            });
            return { content: result.content };
        }
        catch (error) {
            const is403or404 = error instanceof Error && /status (403|404)/.test(error.message);
            if (is403or404) {
                logger.warn("Web fetch failed", getErrorMeta({ url, error }));
            }
            else {
                logger.error("Web fetch failed", getErrorMeta({ url, error }));
            }
            if (error instanceof Error && error.name === "AbortError") {
                throw new Error("Web fetch was aborted");
            }
            if (error instanceof Error) {
                throw new Error(`Failed to fetch URL: ${error.message}`);
            }
            throw new Error("Failed to fetch URL: Unknown error");
        }
    },
    getSummary: (input, output) => {
        const isRedirect = "type" in output &&
            output.type === "redirect" &&
            "redirectUrl" in output;
        if (isRedirect) {
            return {
                input: `url: "${input.url}"`,
                output: `Redirected to ${output.redirectUrl}`,
            };
        }
        return {
            input: `url: "${input.url}"`,
            output: `Processed ${output.content.length} chars`,
        };
    },
}));
//# sourceMappingURL=web-fetch.js.map