import { createLogger } from "../../utils/logger/index.js";
import { init_logger } from "../../utils/logger.js";
import { getDefaultModelSync, init_providers, resolveModel } from "../../providers/index.js";
import { checkUrlSafety, checkWebsiteBlocklist, cleanBase64Images } from "./url-safety.js";
import { complete } from "@earendil-works/pi-ai";
import { Type } from "@sinclair/typebox";
//#region src/agent/tools/web-extract.ts
init_providers();
init_logger();
const log = createLogger("web-extract");
const DEFAULT_WEB_EXTRACT_MAX_LENGTH = 15e3;
const MAX_RAW_HTML_CHARS_FOR_WEB_EXTRACT = 2e5;
const EXTRACTION_TIMEOUT_MS = 3e4;
const MAX_CONTENT_SIZE = 2e6;
const CHUNK_THRESHOLD = 5e5;
const CHUNK_SIZE = 1e5;
const MAX_OUTPUT_SIZE = 15e3;
const WebExtractSchema = Type.Object({
	url: Type.String({ description: "URL to extract content from" }),
	instruction: Type.Optional(Type.String({ description: "What to extract or focus on. Examples: \"main article text\", \"pricing table\", \"API documentation for the auth endpoint\". If omitted, extracts the main content." })),
	maxLength: Type.Optional(Type.Number({ description: "Maximum characters in the extracted content (default: from config or 15000)" }))
});
async function fetchPageContent(url, signal) {
	const response = await fetch(url, {
		signal,
		headers: {
			"User-Agent": "Mozilla/5.0 (compatible; xopc/1.0)",
			Accept: "text/html,application/xhtml+xml,text/plain,application/json"
		},
		redirect: "follow"
	});
	if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}`);
	if ((response.headers.get("content-type") ?? "").includes("application/json")) {
		const json = await response.json();
		return JSON.stringify(json, null, 2);
	}
	return await response.text();
}
function stripHtmlBoilerplate(html, maxRaw = MAX_RAW_HTML_CHARS_FOR_WEB_EXTRACT) {
	let cleaned = html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<svg[\s\S]*?<\/svg>/gi, "").replace(/<noscript[\s\S]*?<\/noscript>/gi, "").replace(/<!--[\s\S]*?-->/g, "");
	cleaned = cleanBase64Images(cleaned);
	cleaned = cleaned.replace(/\s+/g, " ").trim();
	if (cleaned.length > maxRaw) cleaned = cleaned.slice(0, maxRaw) + "\n\n[...truncated]";
	return cleaned;
}
function resolveExtractionModel(config) {
	const envRef = process.env.XOPC_WEB_EXTRACT_MODEL?.trim();
	const configRef = typeof config?.agents?.defaults?.webExtract?.model === "string" ? config.agents.defaults.webExtract.model.trim() : "";
	const ref = envRef || configRef;
	if (ref) try {
		return resolveModel(ref);
	} catch {}
	for (const candidate of ["openai/gpt-4o-mini", "google/gemini-2.0-flash"]) try {
		return resolveModel(candidate);
	} catch {}
	return resolveModel(getDefaultModelSync(config));
}
function buildExtractionSystemPrompt(maxLength) {
	return `You are a web content extractor. Given raw HTML/text from a web page, extract the relevant content as clean, well-structured markdown.

Rules:
- Remove navigation, headers, footers, ads, cookie banners, sidebars
- Preserve the main content structure (headings, lists, tables, code blocks)
- Keep URLs for important links
- If the page has structured data (tables, specs), preserve the structure
- Keep output under ${maxLength} characters\n- If content is truncated, note what was cut
- Output ONLY the extracted content, no meta-commentary`;
}
function buildTimeoutSignal(signal) {
	const timeoutSignal = typeof AbortSignal !== "undefined" && typeof AbortSignal.timeout === "function" ? AbortSignal.timeout(EXTRACTION_TIMEOUT_MS) : void 0;
	if (signal && timeoutSignal && typeof AbortSignal.any === "function") return AbortSignal.any([signal, timeoutSignal]);
	return signal ?? timeoutSignal;
}
function extractTextFromCompletion(result) {
	let text = "";
	if (Array.isArray(result.content)) {
		for (const c of result.content) if (c && typeof c === "object" && c.type === "text") text += String(c.text || "");
	}
	return text.trim();
}
async function callExtractionLlm(systemPrompt, userPrompt, maxTokens, config, signal) {
	return extractTextFromCompletion(await complete(resolveExtractionModel(config), { messages: [{
		role: "user",
		content: `${systemPrompt}\n\n---\n\n${userPrompt}`,
		timestamp: Date.now()
	}] }, {
		maxTokens,
		temperature: .2,
		signal: buildTimeoutSignal(signal)
	}));
}
async function extractWithLlm(rawContent, url, instruction, maxLength, config, signal) {
	const contentLength = rawContent.length;
	if (contentLength > MAX_CONTENT_SIZE) return `[Content too large to process: ${(contentLength / 1e6).toFixed(1)}MB. Try using web_fetch for raw text, or search for a more focused source.]`;
	if (contentLength > CHUNK_THRESHOLD) {
		log.info({
			url,
			contentLength,
			chunkSize: CHUNK_SIZE
		}, "Large document detected, using chunked extraction");
		return extractLargeDocumentChunked(rawContent, url, instruction, maxLength, config, signal);
	}
	const text = await callExtractionLlm(buildExtractionSystemPrompt(maxLength), instruction ? `Extract from this page (${url}):\n\nFOCUS: ${instruction}\n\n---\n\n${rawContent}` : `Extract the main content from this page (${url}):\n\n---\n\n${rawContent}`, Math.min(Math.ceil(maxLength / 3), 8e3), config, signal);
	if (!text) return "No content could be extracted.";
	const outputCap = Math.min(maxLength, MAX_OUTPUT_SIZE);
	if (text.length > outputCap) return text.slice(0, outputCap) + "\n\n[...output truncated to maxLength]";
	log.debug({
		url,
		originalChars: contentLength,
		extractedChars: text.length
	}, "Single-pass extraction complete");
	return text;
}
async function extractLargeDocumentChunked(content, url, instruction, maxLength, config, signal) {
	const chunks = [];
	for (let offset = 0; offset < content.length; offset += CHUNK_SIZE) chunks.push(content.slice(offset, offset + CHUNK_SIZE));
	log.info({
		url,
		chunkCount: chunks.length,
		chunkSize: CHUNK_SIZE
	}, "Split into chunks");
	const chunkPromises = chunks.map(async (chunk, index) => {
		const chunkLabel = `[Chunk ${index + 1}/${chunks.length}]`;
		try {
			const summary = await callExtractionLlm("You are an expert content analyst processing a SECTION of a larger document. Extract and summarize the key information from THIS SECTION ONLY.\n\nGuidelines:\n- Do NOT write introductions or conclusions — this is a partial document\n- Extract ALL key facts, figures, data points, and insights\n- Preserve important quotes, code snippets, and specific details verbatim\n- Use bullet points and structured formatting\n- Note references to other sections without trying to resolve them", `${chunkLabel} Extract key information from this section of ${url}:${instruction ? `\nFOCUS: ${instruction}\n` : ""}\n\n${chunk}`, 6e3, config, signal);
			if (summary) log.debug({
				url,
				chunk: index + 1,
				originalChars: chunk.length,
				summaryChars: summary.length
			}, "Chunk summarized");
			return {
				index,
				summary: summary || null
			};
		} catch (error) {
			const message = error instanceof Error ? error.message : String(error);
			log.warn({
				url,
				chunk: index + 1,
				error: message
			}, "Chunk summarization failed");
			return {
				index,
				summary: null
			};
		}
	});
	const chunkResults = await Promise.all(chunkPromises);
	const summaries = [];
	for (const { index, summary } of chunkResults.sort((a, b) => a.index - b.index)) if (summary) summaries.push(`## Section ${index + 1}\n${summary}`);
	if (summaries.length === 0) {
		const outputCap = Math.min(maxLength, MAX_OUTPUT_SIZE);
		return content.slice(0, outputCap) + "\n\n[...all chunk summaries failed; showing truncated raw content]";
	}
	log.info({
		url,
		successfulChunks: summaries.length,
		totalChunks: chunks.length
	}, "Chunk summaries collected, synthesizing");
	if (summaries.length === 1) {
		const result = summaries[0];
		const outputCap = Math.min(maxLength, MAX_OUTPUT_SIZE);
		return result.length > outputCap ? result.slice(0, outputCap) + "\n\n[...truncated]" : result;
	}
	return synthesizeChunkSummaries(summaries, url, instruction, maxLength, config, signal, content.length);
}
async function synthesizeChunkSummaries(summaries, url, instruction, maxLength, config, signal, originalContentLength) {
	const outputCap = Math.min(maxLength, MAX_OUTPUT_SIZE);
	const systemPrompt = "You synthesize multiple section summaries into one cohesive, comprehensive summary. Remove redundancy, preserve all key facts and actionable information, and organize with clear markdown structure.";
	const userPrompt = `Synthesize these section summaries from ${url} into ONE unified markdown summary (under ${outputCap} characters):${instruction ? `\nFOCUS: ${instruction}\n` : ""}\n\n${summaries.join("\n\n---\n\n")}`;
	try {
		const synthesized = await callExtractionLlm(systemPrompt, userPrompt, 8e3, config, signal);
		if (!synthesized) {
			log.warn({ url }, "Synthesis returned empty, concatenating chunk summaries");
			const fallback = summaries.join("\n\n");
			return fallback.length > outputCap ? fallback.slice(0, outputCap) + "\n\n[...truncated]" : fallback;
		}
		const finalOutput = synthesized.length > outputCap ? synthesized.slice(0, outputCap) + "\n\n[...summary truncated for context management]" : synthesized;
		const compressionRatio = finalOutput.length / originalContentLength;
		log.info({
			url,
			originalChars: originalContentLength,
			extractedChars: finalOutput.length,
			compressionPercent: `${(compressionRatio * 100).toFixed(1)}%`
		}, "Chunked extraction complete");
		return finalOutput;
	} catch (error) {
		const message = error instanceof Error ? error.message : String(error);
		log.warn({
			url,
			error: message
		}, "Synthesis failed, concatenating chunk summaries");
		const fallback = summaries.join("\n\n");
		return fallback.length > outputCap ? fallback.slice(0, outputCap) + "\n\n[...truncated due to synthesis failure]" : fallback;
	}
}
function createWebExtractTool(deps) {
	return {
		name: "web_extract",
		label: "📄 Web Extract",
		description: "Extract and summarize content from a web page using LLM-powered extraction.\n\nUnlike web_fetch (which returns cleaned plain text from HTML), this tool uses a dedicated model pass to produce focused markdown. Use when you need semantic extraction or a specific focus (instruction), not just a single readability pass.\n\nWHEN TO USE:\n- Reading articles, blog posts, documentation\n- Extracting specific data (pricing, specs, API docs)\n- Summarizing long pages to save context\n\nWHEN TO USE web_fetch INSTEAD:\n- Inspecting raw-ish page text quickly without an extra LLM call\n- Downloading JSON/API responses\n- Debugging page rendering issues",
		parameters: WebExtractSchema,
		async execute(_toolCallId, params, signal) {
			const { url, instruction } = params;
			const cfg = deps.getConfig();
			const configuredDefault = cfg?.agents?.defaults?.webExtract?.maxLength ?? 15e3;
			const maxLength = params.maxLength ?? configuredDefault;
			try {
				const safety = checkUrlSafety(url);
				if (!safety.safe) return {
					content: [{
						type: "text",
						text: `Blocked: ${safety.reason}`
					}],
					details: {
						url,
						extractedLength: 0
					}
				};
				const blocked = checkWebsiteBlocklist(url, cfg?.tools?.web?.blocklist);
				if (blocked) return {
					content: [{
						type: "text",
						text: `Blocked: ${blocked.message}`
					}],
					details: {
						url,
						extractedLength: 0
					}
				};
				const cleaned = stripHtmlBoilerplate(await fetchPageContent(url, signal));
				const trimmed = cleaned.trim();
				const looksStructured = trimmed.startsWith("{") || trimmed.startsWith("[");
				if (trimmed.length < 100 && !looksStructured) return {
					content: [{
						type: "text",
						text: `Page at ${url} appears to have no extractable content.`
					}],
					details: {
						url,
						extractedLength: 0
					}
				};
				const extracted = await extractWithLlm(cleaned, url, instruction, maxLength, cfg, signal);
				return {
					content: [{
						type: "text",
						text: extracted
					}],
					details: {
						url,
						extractedLength: extracted.length
					}
				};
			} catch (error) {
				const message = error instanceof Error ? error.message : String(error);
				log.warn({
					url,
					error: message
				}, "web_extract failed");
				return {
					content: [{
						type: "text",
						text: `Failed to extract content from ${url}: ${message}`
					}],
					details: {
						url,
						extractedLength: 0
					}
				};
			}
		}
	};
}
//#endregion
export { DEFAULT_WEB_EXTRACT_MAX_LENGTH, MAX_RAW_HTML_CHARS_FOR_WEB_EXTRACT, createWebExtractTool, stripHtmlBoilerplate };

//# sourceMappingURL=web-extract.js.map