/// <reference path="../../utils/turndown.d.ts" />
/**
 * URL fetch / cache / artifact / markdown-discovery pipeline, mirrored from
 * oh-my-pi's `packages/coding-agent/src/tools/fetch.ts` at 15b5c1397fc.
 *
 * Responsibilities:
 *  - parseReadUrlTarget: URL + line selectors (:raw, :N, :A-B, :A+B, :A-B,C-D),
 *    collapsed-scheme repair, host-port protection.
 *  - session-scoped LRU cache keyed by `<scope>::<raw|rendered>::<normalizedUrl>`,
 *    cached under both requested and final (redirected) URLs.
 *  - renderUrl: markdown discovery (alternate <link>, .md suffix, content
 *    negotiation, llms.txt/llms.md endpoints) + native HTML-to-markdown, with
 *    quality gating.
 *  - artifact persistence of the rendered output, with artifactId surfaced in
 *    truncation metadata when the visible output is head-truncated.
 */
import { lookup } from "node:dns/promises";
import { ipFamily, isPrivateIpAddress, normalizeIpLiteralHost } from "./url-ip-guards.js";
import { LRUCache } from "lru-cache";
import TurndownService from "turndown";
import { Agent, request as undiciRequest } from "undici";
import { getArtifactManager } from "./artifacts.js";
import { extractDocumentMarkdown, isDocumentPath } from "./read-document-extract.js";
export const FETCH_DEFAULT_MAX_LINES = 300;
export const READ_URL_CACHE_MAX_ENTRIES = 100;
const REMOTE_FETCH_TIMEOUT_MS = 10_000;
const MAX_URL_REDIRECTS = 5;
const MAX_URL_RESPONSE_BYTES = 10 * 1024 * 1024;
const MIN_CONTENT_LENGTH = 100;
const readUrlCache = new LRUCache({ max: READ_URL_CACHE_MAX_ENTRIES });
let turndown;
function getTurndown() {
    if (!turndown)
        turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced", bulletListMarker: "-" });
    return turndown;
}
export function repairCollapsedScheme(url) {
    return url.replace(/^(https?):\/([^/])/i, "$1://$2");
}
export function isReadableUrlPath(readPath) {
    const repaired = repairCollapsedScheme(readPath);
    return /^https?:\/\//i.test(repaired) || /^www\./i.test(readPath);
}
function normalizeUrlForCache(url) {
    const repaired = repairCollapsedScheme(url);
    if (/^[a-z][a-z0-9+.-]*:\/\//i.test(repaired))
        return repaired;
    return /^https?:\/\//i.test(repaired) ? repaired : `https://${repaired}`;
}
const RANGE_TOKEN_RE = /^(raw|\d+(?:\+(?:\d+)|(?:-|\.\.)\d+)?)$/i;
const MULTI_RANGE_RE = /^\d+(?:(?:-|\.\.|,|\+)\d+)*$/;
function isUrlSelectorToken(token) {
    return /^raw$/i.test(token) || RANGE_TOKEN_RE.test(token) || MULTI_RANGE_RE.test(token);
}
function parseSingleRange(token) {
    const plus = token.match(/^(\d+)\+(\d+)$/);
    if (plus) {
        const start = Number.parseInt(plus[1], 10), count = Number.parseInt(plus[2], 10);
        return start < 1 || count < 1 ? undefined : { start, end: start + count - 1 };
    }
    const range = token.match(/^(\d+)(?:-|\.\.)(\d+)$/);
    if (range) {
        const start = Number.parseInt(range[1], 10);
        const end = Number.parseInt(range[2], 10);
        return start < 1 || end < start ? undefined : { start, end };
    }
    const single = token.match(/^(\d+)$/);
    if (single) {
        const start = Number.parseInt(single[1], 10);
        return start < 1 ? undefined : { start };
    }
    return undefined;
}
function parseRangeGroup(token) {
    const groups = token.split(",");
    const ranges = [];
    for (const group of groups) {
        const r = parseSingleRange(group);
        if (!r)
            return undefined;
        ranges.push(r);
    }
    return ranges;
}
function tryExtractEmbeddedUrlSelector(readPath) {
    let basePath = readPath;
    const sels = [];
    for (;;) {
        const lastColonIndex = basePath.lastIndexOf(":");
        if (lastColonIndex <= 0)
            break;
        const candidate = basePath.slice(lastColonIndex + 1);
        const remainder = basePath.slice(0, lastColonIndex);
        if (!isReadableUrlPath(remainder) || !isUrlSelectorToken(candidate))
            break;
        try {
            new URL(remainder.startsWith("http://") || remainder.startsWith("https://") ? remainder : `https://${remainder}`);
        }
        catch {
            break;
        }
        sels.unshift(candidate);
        basePath = remainder;
    }
    return sels.length === 0 ? null : { path: basePath, sels };
}
export function parseReadUrlTarget(readPath) {
    const repaired = repairCollapsedScheme(readPath);
    const embedded = tryExtractEmbeddedUrlSelector(repaired);
    const urlPath = embedded?.path ?? repaired;
    if (!isReadableUrlPath(urlPath))
        return null;
    let raw = false;
    let ranges;
    for (const sel of embedded?.sels ?? []) {
        if (/^raw$/i.test(sel)) {
            raw = true;
            continue;
        }
        if (ranges !== undefined)
            throw new Error("URL selector has multiple range groups; combine them with commas (e.g. `:5-10,20-30`).");
        const parsed = parseRangeGroup(sel);
        if (!parsed)
            throw new Error(`Invalid URL line selector: ${sel}`);
        ranges = parsed;
    }
    if (!ranges || ranges.length === 0)
        return { url: urlPath, raw };
    if (ranges.length === 1) {
        const r = ranges[0];
        return { url: urlPath, raw, offset: r.start, limit: r.end !== undefined ? r.end - r.start + 1 : undefined };
    }
    return { url: urlPath, raw, ranges };
}
export function getReadUrlCacheKey(scope, requestedUrl, raw) {
    return `${scope}::${raw ? "raw" : "rendered"}::${normalizeUrlForCache(requestedUrl)}`;
}
function looksLikeHtml(text) {
    return /<html[\s>]/i.test(text) || (/<head[\s>]/i.test(text) && /<body[\s>]/i.test(text));
}
function documentExtensionFromContentType(contentType) {
    if (/pdf/i.test(contentType))
        return ".pdf";
    if (/msword/i.test(contentType))
        return ".doc";
    if (/wordprocessingml|officedocument\.word/i.test(contentType))
        return ".docx";
    if (/presentationml|officedocument\.presentation/i.test(contentType))
        return ".pptx";
    if (/ms-powerpoint|vnd\.ms-powerpoint/i.test(contentType))
        return ".ppt";
    if (/spreadsheetml|officedocument\.spreadsheet/i.test(contentType))
        return ".xlsx";
    if (/ms-excel|vnd\.ms-excel/i.test(contentType))
        return ".xls";
    if (/epub/i.test(contentType))
        return ".epub";
    if (/\bjson\b|ipynb|jupyter/i.test(contentType))
        return ".ipynb";
    if (/rtf/i.test(contentType))
        return ".rtf";
    return undefined;
}
function createPinnedDispatcher(pinned) {
    return new Agent({ connect: { lookup: (_hostname, options, callback) => { if (options.all)
                callback(null, [pinned]);
            else
                callback(null, pinned.address, pinned.family); } } });
}
async function closePinnedDispatcher(dispatcher) {
    const close = dispatcher?.close;
    if (typeof close === "function")
        await close.call(dispatcher);
}
/**
 * Fetch a URL through undici with the pinned dispatcher so the validated
 * address is the one actually connected to. Unlike `globalThis.fetch`, which
 * silently ignores the undici `dispatcher` option under Bun's compiled binary
 * (reopening a DNS-rebinding TOCTOU window for hostname targets), undici's own
 * client honors the dispatcher's custom `lookup` regardless of host runtime.
 * Adapts the undici response to the WHATWG-Response shape loadPage consumes.
 */
async function pinnedFetch(url, dispatcher, options) {
    const result = await undiciRequest(url, { method: "GET", dispatcher, signal: options.signal, headers: options.headers });
    const body = result.body;
    const status = result.statusCode;
    const headers = new Headers();
    for (const [key, value] of Object.entries(result.headers)) {
        if (value === undefined)
            continue;
        const values = Array.isArray(value) ? value : [value];
        for (const entry of values)
            headers.append(key, entry);
    }
    const stream = new ReadableStream({
        start(controller) { body.on("data", (chunk) => controller.enqueue(new Uint8Array(chunk))); body.on("end", () => controller.close()); body.on("error", (error) => controller.error(error)); },
        cancel() { body.destroy(); },
    });
    return new Response(stream, { status, headers });
}
async function assertSafeFetchUrl(url) {
    const parsed = new URL(url);
    if (parsed.protocol !== "http:" && parsed.protocol !== "https:")
        throw new Error(`Unsupported URL protocol: ${parsed.protocol}`);
    if (process.env.ATOMIC_ALLOW_PRIVATE_URL_READS === "1")
        return undefined;
    const hostname = parsed.hostname.replace(/^\[|\]$/g, "").replace(/\.$/, "").toLowerCase();
    if (hostname === "localhost" || hostname.endsWith(".localhost") || hostname === "metadata.google.internal")
        throw new Error(`Refusing to fetch private or metadata URL: ${url}`);
    const literalAddress = normalizeIpLiteralHost(hostname);
    if (literalAddress) {
        if (isPrivateIpAddress(literalAddress))
            throw new Error(`Refusing to fetch private or metadata URL: ${url}`);
        return { address: literalAddress, family: ipFamily(literalAddress), pinned: false };
    }
    const addresses = await lookup(hostname, { all: true }).catch((error) => { throw new Error(`Could not resolve URL host ${hostname}: ${error instanceof Error ? error.message : String(error)}`); });
    if (addresses.length === 0)
        throw new Error(`Could not resolve URL host ${hostname}: no addresses returned`);
    for (const address of addresses)
        if (isPrivateIpAddress(address.address))
            throw new Error(`Refusing to fetch private or metadata URL: ${url}`);
    const first = addresses[0];
    return { address: first.address, family: first.family === 6 ? 6 : 4, pinned: true };
}
async function readResponseTextWithLimit(response) {
    if (!response.body) {
        const buffer = Buffer.from(await response.arrayBuffer());
        if (buffer.length > MAX_URL_RESPONSE_BYTES)
            throw new Error(`URL response exceeds ${MAX_URL_RESPONSE_BYTES} byte limit`);
        return buffer.toString("utf8");
    }
    const reader = response.body.getReader();
    const decoder = new TextDecoder();
    const chunks = [];
    let totalBytes = 0;
    for (;;) {
        const { done, value } = await reader.read();
        if (done)
            break;
        totalBytes += value.byteLength;
        if (totalBytes > MAX_URL_RESPONSE_BYTES) {
            await reader.cancel();
            throw new Error(`URL response exceeds ${MAX_URL_RESPONSE_BYTES} byte limit`);
        }
        chunks.push(decoder.decode(value, { stream: true }));
    }
    chunks.push(decoder.decode());
    return chunks.join("");
}
export async function loadPage(url, timeoutMs, signal, accept) {
    url = normalizeUrlForCache(url);
    const controller = new AbortController();
    const timer = setTimeout(() => controller.abort(), Math.min(timeoutMs, REMOTE_FETCH_TIMEOUT_MS));
    const onParentAbort = () => controller.abort();
    signal?.addEventListener("abort", onParentAbort, { once: true });
    try {
        let currentUrl = url;
        for (let redirects = 0; redirects <= MAX_URL_REDIRECTS; redirects++) {
            const pinned = await assertSafeFetchUrl(currentUrl);
            // Only a DNS-resolved hostname needs pinning: a literal-IP host is
            // validated directly and cannot be re-resolved at connect time, so
            // global fetch is sufficient and stays redirect/mock-friendly.
            const dispatcher = pinned?.pinned ? createPinnedDispatcher(pinned) : undefined;
            try {
                const headers = accept ? { Accept: accept } : undefined;
                // Route pinned (hostname) fetches through undici directly so the
                // dispatcher's custom lookup is honored even under Bun's compiled
                // fetch, which silently ignores the `dispatcher` option.
                const response = dispatcher
                    ? await pinnedFetch(currentUrl, dispatcher, { signal: controller.signal, headers })
                    : await fetch(currentUrl, { signal: controller.signal, headers, redirect: "manual" });
                if (response.status >= 300 && response.status < 400 && response.headers.get("location")) {
                    await response.body?.cancel().catch(() => { });
                    currentUrl = new URL(response.headers.get("location"), currentUrl).href;
                    continue;
                }
                const contentLength = Number.parseInt(response.headers.get("content-length") ?? "0", 10);
                if (contentLength > MAX_URL_RESPONSE_BYTES)
                    throw new Error(`URL response exceeds ${MAX_URL_RESPONSE_BYTES} byte limit: ${currentUrl}`);
                const contentType = response.headers.get("content-type") ?? "";
                const content = await readResponseTextWithLimit(response);
                return { ok: response.ok, status: response.status, content, contentType, finalUrl: response.url || currentUrl };
            }
            finally {
                await closePinnedDispatcher(dispatcher).catch(() => undefined);
            }
        }
        throw new Error(`Too many redirects while fetching URL: ${url}`);
    }
    finally {
        clearTimeout(timer);
        signal?.removeEventListener("abort", onParentAbort);
    }
}
function isLowQuality(content) {
    const lower = content.toLowerCase();
    if (/(enable javascript|javascript required|turn on javascript|please enable javascript|browser not supported)/.test(lower) && content.trim().length < 2000)
        return true;
    const nonblank = content.split("\n").filter((line) => line.trim().length > 0);
    if (nonblank.length > 10 && nonblank.filter((line) => line.trim().length < 40).length / nonblank.length > 0.7)
        return true;
    return false;
}
function buildLlmEndpointCandidates(url) {
    const parsed = (() => { try {
        return new URL(url);
    }
    catch {
        return undefined;
    } })();
    if (!parsed)
        return [];
    const origin = parsed.origin;
    if (parsed.pathname === "/" || parsed.pathname === "")
        return [`${origin}/.well-known/llms.txt`, `${origin}/llms.txt`, `${origin}/llms.md`];
    const trimmed = parsed.pathname.replace(/\/+$/, "");
    const segments = trimmed.split("/").filter(Boolean);
    const candidates = [];
    const depth = parsed.pathname.endsWith("/") ? segments.length : Math.max(segments.length - 1, 1);
    for (let scope = Math.min(depth, segments.length); scope >= 1; scope--) {
        const base = `${origin}/${segments.slice(0, scope).join("/")}`;
        candidates.push(`${base}/llms.txt`, `${base}/llms.md`);
    }
    candidates.push(`${origin}/llms.txt`, `${origin}/llms.md`);
    return [...new Set(candidates)];
}
async function tryLlmEndpoints(url, signal) {
    for (const endpoint of buildLlmEndpointCandidates(url)) {
        try {
            const page = await loadPage(endpoint, 5000, signal);
            if (page.ok && page.content.trim().length > MIN_CONTENT_LENGTH && !looksLikeHtml(page.content))
                return { content: page.content, endpoint };
        }
        catch { }
    }
    return null;
}
function parseAlternateMarkdownLink(html, pageUrl) {
    const headEnd = html.toLowerCase().indexOf("</head>");
    const head = html.slice(0, headEnd >= 0 ? headEnd + 7 : html.length);
    const links = [...head.matchAll(/<link\b[^>]*>/gi)].map((m) => m[0] ?? "");
    for (const tag of links) {
        const rel = /rel=["']?([^"'>]+)["']?/i.exec(tag)?.[1] ?? "";
        const type = /type=["']?([^"'>]+)["']?/i.exec(tag)?.[1] ?? "";
        const href = /href=["']?([^"'>]+)["']?/i.exec(tag)?.[1] ?? "";
        if (!/alternate/i.test(rel) || !href)
            continue;
        if (/RecentChanges|Special:|\/feed\/|action=feed/i.test(href))
            continue;
        if (type.includes("markdown") || href.endsWith(".md")) {
            try {
                return new URL(href, pageUrl).href;
            }
            catch { }
        }
    }
    return undefined;
}
async function tryMdSuffix(url, signal) {
    const candidates = [];
    if (/\/$/.test(url))
        candidates.push(`${url}index.html.md`);
    else if (/\.\w+\/?$/.test(url))
        candidates.push(`${url}.md`);
    else
        candidates.push(`${url}.md`, `${url}/index.html.md`);
    for (const candidate of candidates) {
        try {
            const page = await loadPage(candidate, 5000, signal);
            if (page.ok && page.content.trim().length > MIN_CONTENT_LENGTH && !looksLikeHtml(page.content))
                return page.content;
        }
        catch { }
    }
    return null;
}
async function tryContentNegotiation(url, signal) {
    try {
        const page = await loadPage(url, 5000, signal, "text/markdown, text/plain;q=0.9, text/html;q=0.8");
        const mime = page.contentType.toLowerCase();
        if (page.ok && page.content.trim().length > MIN_CONTENT_LENGTH && !looksLikeHtml(page.content) && (mime.includes("markdown") || mime.includes("text/plain")))
            return page.content;
    }
    catch { }
    return null;
}
export async function renderUrl(url, raw, signal) {
    const notes = [];
    let method = "text";
    let contentType = "";
    let finalUrl;
    let body = "";
    const page = await loadPage(url, REMOTE_FETCH_TIMEOUT_MS, signal);
    finalUrl = page.finalUrl;
    contentType = page.contentType;
    if (!page.ok)
        return { content: page.content || `(request failed with status ${page.status})`, finalUrl, contentType, method: "failed", notes, truncated: false };
    if (raw)
        return { content: page.content, finalUrl, contentType, method: "raw", notes, truncated: false };
    const buffer = Buffer.from(page.content, "utf8");
    const docExt = documentExtensionFromContentType(contentType) ?? (isDocumentPath(finalUrl) ? (finalUrl.match(/\.\w+(?:$|[?#])/)?.[0] ?? "") : "");
    if (docExt) {
        try {
            body = await extractDocumentMarkdown(buffer, `${finalUrl}${docExt}`);
            method = "document";
        }
        catch (error) {
            body = `[Cannot read ${docExt} file: ${error instanceof Error ? error.message : "conversion failed"}]`;
            method = "document-error";
        }
        return { content: body, finalUrl, contentType, method, notes, truncated: false };
    }
    const isHtml = /html/i.test(contentType) || looksLikeHtml(page.content);
    if (isHtml) {
        const altMd = parseAlternateMarkdownLink(page.content, finalUrl);
        if (altMd) {
            try {
                const alt = await loadPage(altMd, 5000, signal);
                if (alt.ok && alt.content.trim().length > MIN_CONTENT_LENGTH && !looksLikeHtml(alt.content)) {
                    return { content: alt.content, finalUrl: alt.finalUrl, contentType: alt.contentType, method: "alternate-markdown", notes, truncated: false };
                }
            }
            catch { }
            notes.push("alternate markdown link unusable");
        }
        const md = await tryMdSuffix(finalUrl, signal);
        if (md)
            return { content: md, finalUrl, contentType: "text/markdown", method: "md-suffix", notes, truncated: false };
        const neg = await tryContentNegotiation(finalUrl, signal);
        if (neg)
            return { content: neg, finalUrl, contentType: "text/markdown", method: "content-negotiation", notes, truncated: false };
        let rendered = getTurndown().turndown(page.content).trim();
        method = "native";
        if (rendered.length <= MIN_CONTENT_LENGTH || isLowQuality(rendered)) {
            const llms = await tryLlmEndpoints(finalUrl, signal);
            if (llms) {
                return { content: llms.content, finalUrl, contentType: "text/plain", method: `llms-txt (${llms.endpoint})`, notes, truncated: false };
            }
            notes.push("rendered content low quality; no markdown source discovered");
        }
        body = rendered;
    }
    else {
        body = page.content;
    }
    return { content: body, finalUrl, contentType, method, notes, truncated: false };
}
export function buildUrlReadOutput(result) {
    const header = [`URL: ${result.finalUrl}`, `Content-Type: ${result.contentType}`, `Method: ${result.method}`, ...(result.notes.length ? [`Notes: ${result.notes.join("; ")}`] : [])].join("\n");
    return `${header}\n\n---\n\n${result.content}`;
}
function persistReadUrlArtifact(artifactsDir, output) {
    if (!artifactsDir)
        return undefined;
    return getArtifactManager(artifactsDir).save(output, "read");
}
/**
 * Resolve a URL to its full rendered (or raw) output, using the session-scoped
 * cache and persisting the rendered body as a `read` artifact. Returns the
 * complete untruncated output so callers can apply their own line selection.
 */
export async function loadReadUrlOutput(scope, params, artifactsDir, signal) {
    const parsed = parseReadUrlTarget(params.path);
    if (!parsed)
        throw new Error(`Not a readable URL: ${params.path}`);
    const raw = params.raw ?? parsed.raw;
    const key = getReadUrlCacheKey(scope, parsed.url, raw);
    let entry = readUrlCache.get(key);
    if (!entry) {
        const rendered = await renderUrl(parsed.url, raw, signal);
        const output = raw ? rendered.content : buildUrlReadOutput(rendered);
        const artifactId = persistReadUrlArtifact(artifactsDir, output);
        const details = { url: parsed.url, finalUrl: rendered.finalUrl, contentType: rendered.contentType, method: rendered.method, truncated: rendered.truncated, notes: rendered.notes, meta: artifactId ? { artifactId } : undefined };
        entry = { artifactId, details, output };
        readUrlCache.set(key, entry);
        readUrlCache.set(getReadUrlCacheKey(scope, rendered.finalUrl, raw), entry);
    }
    return { output: entry.output, details: entry.details, artifactId: entry.artifactId, raw };
}
export async function executeReadUrl(scope, params, artifactsDir, signal) {
    const loaded = await loadReadUrlOutput(scope, params, artifactsDir, signal);
    const entry = { output: loaded.output, details: loaded.details, artifactId: loaded.artifactId };
    const fetchDefaultBytes = 50 * 1024;
    const lines = entry.output.split("\n");
    const visibleLines = lines.slice(0, FETCH_DEFAULT_MAX_LINES);
    let content = visibleLines.join("\n");
    const bytesBeforeByteClamp = Buffer.byteLength(content, "utf8");
    const byteTruncated = bytesBeforeByteClamp > fetchDefaultBytes;
    const lineTruncated = lines.length > FETCH_DEFAULT_MAX_LINES;
    if (byteTruncated)
        content = content.slice(0, fetchDefaultBytes);
    const truncated = lineTruncated || byteTruncated;
    let artifactId = entry.artifactId;
    if (truncated && !artifactId)
        artifactId = persistReadUrlArtifact(artifactsDir, entry.output);
    const outputLines = content.length === 0 ? 0 : content.split("\n").length;
    const truncation = truncated ? {
        content,
        truncated: true,
        truncatedBy: byteTruncated ? "bytes" : "lines",
        totalLines: lines.length,
        totalBytes: Buffer.byteLength(entry.output, "utf8"),
        outputLines,
        outputBytes: Buffer.byteLength(content, "utf8"),
        lastLinePartial: byteTruncated,
        firstLineExceedsLimit: false,
        maxLines: FETCH_DEFAULT_MAX_LINES,
        maxBytes: fetchDefaultBytes,
    } : undefined;
    const details = { ...entry.details, truncated, meta: { ...(entry.details.meta ?? {}), ...(artifactId ? { artifactId } : {}), ...(truncation ? { truncation } : {}) } };
    if (truncated)
        content += `\n\n[Showing first ${visibleLines.length} of ${lines.length} lines.${artifactId ? ` Full output: artifact://${artifactId}` : ""}]`;
    return { content, details, artifactId };
}
export function resetReadUrlCache() {
    readUrlCache.clear();
}
//# sourceMappingURL=fetch-url.js.map