/**
 * fetch_page tool — Extract clean markdown from any URL.
 *
 * v3 improvements:
 * - offset parameter for continuation reading (like file read offsets)
 * - selector parameter for Jina's X-Target-Selector (extract specific sections)
 * - Jina failure diagnostics surfaced in details
 * - Content-type awareness (JSON passthrough, PDF detection)
 */
import { truncateHead, DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES } from "@nerf/pi-coding-agent";
import { Text } from "@nerf/pi-tui";
import { Type } from "@sinclair/typebox";
import { LRUTTLCache } from "./cache.js";
import { fetchSimple, HttpError } from "./http.js";
import { extractDomain, isBlockedUrl } from "./url-utils.js";
import { formatPageContent } from "./format.js";
import { getOllamaApiKey } from "./provider.js";
// Page content cache: max 30 entries, 15-minute TTL
const pageCache = new LRUTTLCache({ max: 30, ttlMs: 900_000 });
pageCache.startPurgeInterval(120_000);
// =============================================================================
// Jina Reader
// =============================================================================
/**
 * Fetch page content via Jina Reader API.
 * Returns content + metadata, or throws with a descriptive error.
 */
async function fetchViaJina(url, options = {}) {
    const jinaUrl = `https://r.jina.ai/${url}`;
    const headers = {
        "Accept": "text/plain",
        "X-Return-Format": "markdown",
        "X-No-Cache": "false",
    };
    // Use Jina API key if available for higher rate limits
    const jinaKey = process.env.JINA_API_KEY;
    if (jinaKey) {
        headers["Authorization"] = `Bearer ${jinaKey}`;
    }
    // Target specific CSS selector on the page
    if (options.selector) {
        headers["X-Target-Selector"] = options.selector;
    }
    const response = await fetchSimple(jinaUrl, {
        method: "GET",
        headers,
        signal: options.signal,
        timeoutMs: 20_000,
    });
    const text = await response.text();
    // Jina returns markdown with a title line at the top
    // Format: "Title: <title>\nURL Source: <url>\n\n<content>"
    let title;
    let content = text;
    const titleMatch = text.match(/^Title:\s*(.+)\n/);
    if (titleMatch) {
        title = titleMatch[1].trim();
        content = text.replace(/^Title:\s*.+\n/, "");
    }
    // Strip the URL Source line
    content = content.replace(/^URL Source:\s*.+\n\n?/, "");
    // Strip Markdown images to save tokens
    content = content.replace(/!\[([^\]]*)\]\([^)]+\)/g, "");
    // Collapse excessive whitespace
    content = content.replace(/\n{4,}/g, "\n\n\n");
    return { content: content.trim(), title };
}
/**
 * Basic fallback: fetch raw HTML and do crude text extraction.
 */
async function fetchDirectFallback(url, signal) {
    const response = await fetchSimple(url, {
        method: "GET",
        headers: {
            "Accept": "text/html,application/xhtml+xml,application/json,text/plain",
            "User-Agent": "Mozilla/5.0 (compatible; pi-coding-agent/1.0)",
        },
        signal,
        timeoutMs: 15_000,
    });
    const contentType = response.headers.get("content-type") || "";
    // JSON passthrough — return formatted JSON directly
    if (contentType.includes("application/json")) {
        const text = await response.text();
        try {
            const parsed = JSON.parse(text);
            return {
                content: "```json\n" + JSON.stringify(parsed, null, 2) + "\n```",
                title: undefined,
                contentType: "application/json",
            };
        }
        catch {
            return { content: text, title: undefined, contentType };
        }
    }
    // Plain text passthrough
    if (contentType.includes("text/plain")) {
        const text = await response.text();
        return { content: text, title: undefined, contentType: "text/plain" };
    }
    // PDF detection — can't extract, but tell the agent
    if (contentType.includes("application/pdf")) {
        return {
            content: "[This URL is a PDF document. Content extraction is not supported for PDFs.]",
            title: undefined,
            contentType: "application/pdf",
        };
    }
    const html = await response.text();
    // Extract title
    const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
    const title = titleMatch ? titleMatch[1].trim() : undefined;
    // Strip tags, decode entities, collapse whitespace
    let text = html
        .replace(/<script[\s\S]*?<\/script>/gi, "")
        .replace(/<style[\s\S]*?<\/style>/gi, "")
        .replace(/<nav[\s\S]*?<\/nav>/gi, "")
        .replace(/<header[\s\S]*?<\/header>/gi, "")
        .replace(/<footer[\s\S]*?<\/footer>/gi, "")
        .replace(/<\/?(p|div|br|h[1-6]|li|tr|blockquote|pre|section|article)[^>]*>/gi, "\n")
        .replace(/<[^>]+>/g, " ")
        .replace(/&amp;/g, "&")
        .replace(/&lt;/g, "<")
        .replace(/&gt;/g, ">")
        .replace(/&quot;/g, '"')
        .replace(/&#39;/g, "'")
        .replace(/&nbsp;/g, " ")
        .replace(/[ \t]+/g, " ")
        .replace(/\n[ \t]+/g, "\n")
        .replace(/\n{3,}/g, "\n\n")
        .trim();
    return { content: text, title, contentType };
}
/**
 * Fetch page content via Ollama web_fetch API.
 * Returns content + metadata, or throws on failure.
 */
async function fetchViaOllama(url, signal) {
    const response = await fetchSimple("https://ollama.com/api/web_fetch", {
        method: "POST",
        headers: {
            "Content-Type": "application/json",
            "Authorization": `Bearer ${getOllamaApiKey()}`,
        },
        body: JSON.stringify({ url }),
        signal,
        timeoutMs: 20_000,
    });
    const data = await response.json();
    const content = (data.content || "").trim();
    const title = data.title?.trim() || undefined;
    return { content, title };
}
// =============================================================================
// Smart Truncation
// =============================================================================
/**
 * Truncate page content to a target character count, trying to break
 * at paragraph boundaries rather than mid-sentence.
 */
function smartTruncate(content, maxChars, offset = 0) {
    // Apply offset first
    const sliced = offset > 0 ? content.slice(offset) : content;
    if (sliced.length <= maxChars) {
        return { content: sliced, truncated: false, hasMore: false };
    }
    // Find the last paragraph break before maxChars
    const window = sliced.slice(0, maxChars);
    const lastParagraph = window.lastIndexOf("\n\n");
    const lastSentence = window.lastIndexOf(". ");
    const lastNewline = window.lastIndexOf("\n");
    // Prefer paragraph > sentence > newline > hard cut
    let cutPoint = maxChars;
    if (lastParagraph > maxChars * 0.6) {
        cutPoint = lastParagraph;
    }
    else if (lastSentence > maxChars * 0.6) {
        cutPoint = lastSentence + 1;
    }
    else if (lastNewline > maxChars * 0.6) {
        cutPoint = lastNewline;
    }
    const nextOffset = offset + cutPoint;
    const hasMore = nextOffset < content.length;
    return {
        content: sliced.slice(0, cutPoint).trim() + "\n\n[... content truncated]",
        truncated: true,
        hasMore,
        nextOffset: hasMore ? nextOffset : undefined,
    };
}
async function fetchOnePage(url, options) {
    let pageContent;
    let pageTitle;
    let source = "jina";
    let jinaError;
    let contentType;
    try {
        const result = await fetchViaJina(url, options);
        pageContent = result.content;
        pageTitle = result.title;
    }
    catch (err) {
        // Capture Jina failure reason for diagnostics
        jinaError = err instanceof HttpError
            ? `Jina HTTP ${err.statusCode}`
            : err.message ?? String(err);
        // Try Ollama web_fetch as intermediate fallback if API key is available
        const ollamaKey = getOllamaApiKey();
        if (ollamaKey) {
            try {
                const ollamaResult = await fetchViaOllama(url, options.signal);
                if (ollamaResult.content && ollamaResult.content.length >= 50) {
                    pageContent = ollamaResult.content;
                    pageTitle = ollamaResult.title;
                    source = "direct";
                    return {
                        content: pageContent,
                        title: pageTitle,
                        source,
                        jinaError,
                        contentType,
                        originalChars: pageContent.length,
                    };
                }
            }
            catch {
                // Ollama fetch failed too — fall through to direct
            }
        }
        source = "direct";
        const result = await fetchDirectFallback(url, options.signal);
        pageContent = result.content;
        pageTitle = result.title;
        contentType = result.contentType;
    }
    return {
        content: pageContent,
        title: pageTitle,
        source,
        jinaError,
        contentType,
        originalChars: pageContent.length,
    };
}
// =============================================================================
// Tool Registration
// =============================================================================
export function registerFetchPageTool(pi) {
    pi.registerTool({
        name: "fetch_page",
        label: "Fetch Page",
        description: "Fetch a web page and extract its content as clean markdown. " +
            "Use this to read the full content of URLs found via search-the-web. " +
            "Uses Jina Reader for high-quality markdown extraction. " +
            "Control the amount of content returned with maxChars (default: 8000, max: 30000).",
        promptSnippet: "Fetch and extract clean content from a web page URL as markdown",
        promptGuidelines: [
            "Use fetch_page to read the content of URLs found via search-the-web when you need more detail than snippets provide.",
            "Start with the default maxChars (8000). Increase only if the first fetch lacks the detail you need.",
            "For very long pages, use a smaller maxChars and increase if needed — this saves context tokens.",
            "The extracted content is already clean markdown — no HTML tags, no navigation, no ads.",
        ],
        parameters: Type.Object({
            url: Type.String({ description: "URL to fetch and extract content from" }),
            maxChars: Type.Optional(Type.Number({
                minimum: 1000,
                maximum: 30000,
                default: 8000,
                description: "Maximum characters of content to return (default: 8000, max: 30000). Controls context token usage.",
            })),
            offset: Type.Optional(Type.Number({
                minimum: 0,
                description: "Character offset to start reading from (for continuation of truncated pages). Use the nextOffset value from a previous fetch_page result.",
            })),
            selector: Type.Optional(Type.String({
                description: "CSS selector to extract only a specific section of the page (e.g., 'main', 'article', '.api-docs'). Reduces noise and token usage.",
            })),
        }),
        async execute(toolCallId, params, signal, onUpdate, ctx) {
            if (signal?.aborted) {
                return { content: [{ type: "text", text: "Fetch cancelled." }], details: undefined };
            }
            const maxChars = params.maxChars ?? 8000;
            const offset = params.offset ?? 0;
            const url = params.url.trim();
            // Validate URL
            try {
                new URL(url);
            }
            catch {
                return {
                    content: [{ type: "text", text: `Invalid URL: ${url}` }],
                    isError: true,
                    details: { error: "Invalid URL", url },
                };
            }
            if (isBlockedUrl(url)) {
                return {
                    content: [{ type: "text", text: `Blocked URL: requests to private/internal addresses are not allowed.` }],
                    isError: true,
                    details: { error: "SSRF blocked", url },
                };
            }
            // ------------------------------------------------------------------
            // Cache lookup (full content cached, offset/truncation applied after)
            // ------------------------------------------------------------------
            const cacheKey = params.selector ? `${url}|sel:${params.selector}` : url;
            const cached = pageCache.get(cacheKey);
            if (cached) {
                const trunc = smartTruncate(cached.content, maxChars, offset);
                const opts = {
                    title: cached.title,
                    charCount: trunc.content.length,
                    truncated: trunc.truncated,
                    originalChars: trunc.truncated ? cached.content.length : undefined,
                    hasMore: trunc.hasMore,
                    nextOffset: trunc.nextOffset,
                };
                const output = formatPageContent(url, trunc.content, opts);
                const finalTruncation = truncateHead(output, { maxLines: DEFAULT_MAX_LINES, maxBytes: DEFAULT_MAX_BYTES });
                const details = {
                    url,
                    title: cached.title,
                    charCount: trunc.content.length,
                    originalChars: cached.content.length,
                    truncated: trunc.truncated,
                    cached: true,
                    source: cached.source,
                    hasMore: trunc.hasMore,
                    nextOffset: trunc.nextOffset,
                    offset: offset || undefined,
                };
                return {
                    content: [{ type: "text", text: finalTruncation.content }],
                    details,
                };
            }
            const domain = extractDomain(url);
            onUpdate?.({ content: [{ type: "text", text: `Fetching ${domain}...` }], details: undefined });
            // ------------------------------------------------------------------
            // Fetch page content
            // ------------------------------------------------------------------
            let result;
            try {
                result = await fetchOnePage(url, { signal, selector: params.selector });
            }
            catch (err) {
                const message = err instanceof HttpError
                    ? `HTTP ${err.statusCode}`
                    : err.message ?? String(err);
                return {
                    content: [{ type: "text", text: `Failed to fetch ${domain}: ${message}` }],
                    isError: true,
                    details: { error: message, url },
                };
            }
            // Check for empty content
            if (!result.content || result.content.length < 50) {
                return {
                    content: [{ type: "text", text: `Page at ${domain} returned no extractable content.` }],
                    details: { url, charCount: 0, source: result.source, cached: false, truncated: false, jinaError: result.jinaError },
                };
            }
            // Cache the full content
            pageCache.set(cacheKey, { content: result.content, title: result.title, source: result.source });
            // Smart truncate with offset
            const trunc = smartTruncate(result.content, maxChars, offset);
            const opts = {
                title: result.title,
                charCount: trunc.content.length,
                truncated: trunc.truncated,
                originalChars: trunc.truncated ? result.originalChars : undefined,
                hasMore: trunc.hasMore,
                nextOffset: trunc.nextOffset,
            };
            const output = formatPageContent(url, trunc.content, opts);
            const finalTruncation = truncateHead(output, { maxLines: DEFAULT_MAX_LINES, maxBytes: DEFAULT_MAX_BYTES });
            let content = finalTruncation.content;
            if (finalTruncation.truncated) {
                const tempFile = await pi.writeTempFile(output, { prefix: "fetch-page-" });
                content += `\n\n[Truncated to fit context. Full content: ${tempFile}]`;
            }
            const details = {
                url,
                title: result.title,
                charCount: trunc.content.length,
                originalChars: result.originalChars,
                truncated: trunc.truncated,
                cached: false,
                source: result.source,
                jinaError: result.jinaError,
                contentType: result.contentType,
                hasMore: trunc.hasMore,
                nextOffset: trunc.nextOffset,
                offset: offset || undefined,
                selector: params.selector,
            };
            return {
                content: [{ type: "text", text: content }],
                details,
            };
        },
        renderCall(args, theme) {
            const domain = extractDomain(args.url);
            let text = theme.fg("toolTitle", theme.bold("fetch_page "));
            text += theme.fg("accent", domain);
            const meta = [];
            if (args.maxChars && args.maxChars !== 8000)
                meta.push(`max ${(args.maxChars / 1000).toFixed(0)}k`);
            if (args.offset)
                meta.push(`offset:${args.offset}`);
            if (args.selector)
                meta.push(`sel:"${args.selector}"`);
            if (meta.length > 0) {
                text += " " + theme.fg("dim", `(${meta.join(", ")})`);
            }
            return new Text(text, 0, 0);
        },
        renderResult(result, { expanded }, theme) {
            const details = result.details;
            if (details?.error) {
                return new Text(theme.fg("error", `✗ ${details.error}`), 0, 0);
            }
            const domain = extractDomain(details?.url || "");
            const title = details?.title ? ` — ${details.title}` : "";
            const chars = details?.charCount ? `${(details.charCount / 1000).toFixed(1)}k chars` : "";
            const cacheTag = details?.cached ? theme.fg("dim", " [cached]") : "";
            const sourceTag = details?.source === "direct" ? theme.fg("dim", " [direct]") : "";
            const truncTag = details?.truncated && details?.originalChars
                ? theme.fg("dim", ` [${(details.originalChars / 1000).toFixed(0)}k total]`)
                : "";
            const moreTag = details?.hasMore && details?.nextOffset
                ? theme.fg("accent", ` [more→offset:${details.nextOffset}]`)
                : "";
            const jinaTag = details?.jinaError
                ? theme.fg("warning", ` [jina failed: ${details.jinaError}]`)
                : "";
            let text = theme.fg("success", `✓ ${domain}${title}`) + ` ${chars}` +
                cacheTag + sourceTag + truncTag + moreTag + jinaTag;
            if (expanded) {
                const content = result.content[0];
                if (content?.type === "text") {
                    const preview = content.text.split("\n").slice(0, 8).join("\n");
                    text += "\n\n" + theme.fg("dim", preview);
                }
            }
            return new Text(text, 0, 0);
        },
    });
}
