// url-get — faithful page retrieval, no model in the path.
//
//   FETCH ─▶ HTTP STATUS ─▶ CONTENT-TYPE ─▶ CHALLENGE/EMPTY ─▶ HTML→MD ─▶ WRITE REF ─▶ RETURN TEXT
//     │          │              │                │              │           │            │
//     ▼          ▼              ▼                ▼              ▼           ▼            ▼
//  [network?] [non-2xx?]   [non-html?]    [challenge?empty?]  (verbatim)  (ground    (capped
//   error=     error=        error=          error=                       truth on    cleaned md,
//   fetch      http          non-html        challenge|empty              disk first) no inference)
//
// One mechanism, two surfaces of one artefact: the VERBATIM markdown (no model
// in the path — nothing to refuse on copyright) written to an account-scoped
// reference file, and the same cleaned text returned inline, capped. Only
// { text, referencePath } reaches the caller. There is no subprocess inference:
// a caller that needs a summary delegates to a subagent which holds the full
// text in its own context and returns only the summary to the main agent.
import { createHash } from "node:crypto";
import { mkdir, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { NodeHtmlMarkdown } from "node-html-markdown";
const FETCH_TIMEOUT_MS = 30_000;
const TEXT_INPUT_CAP = 100_000;
const USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " +
    "(KHTML, like Gecko) Chrome/124.0 Safari/537.36";
// Strong, low-false-positive markers of a bot-challenge interstitial. These
// strings do not occur in legitimate article bodies; matching one means the
// response is a challenge page, not the requested content.
const CHALLENGE_SIGNATURES = [
    "/cdn-cgi/challenge-platform/",
    "cf-browser-verification",
    "enable javascript and cookies to continue",
    "checking your browser before accessing",
    "attention required! | cloudflare",
    "<title>just a moment",
];
/** Resolve the account-scoped storage root. ACCOUNT_DIR is the canonical env
 *  var (set on the spawned claude process); fall back to deriving it from
 *  PLATFORM_ROOT + ACCOUNT_ID so the tool still works when only those are
 *  present. Returns null when neither is resolvable — the caller turns that
 *  into an error envelope rather than throwing. */
function resolveAccountDir() {
    const direct = process.env.ACCOUNT_DIR;
    if (direct && direct.trim())
        return direct;
    const platformRoot = process.env.PLATFORM_ROOT;
    const accountId = process.env.ACCOUNT_ID;
    if (platformRoot && accountId && !accountId.includes("/") && !accountId.includes("..")) {
        return join(platformRoot, "..", "data", "accounts", accountId);
    }
    return null;
}
function isChallenge(html) {
    const haystack = html.slice(0, 8000).toLowerCase();
    return CHALLENGE_SIGNATURES.some((sig) => haystack.includes(sig));
}
export async function urlGet(params) {
    const { url } = params;
    const t0 = Date.now();
    let res;
    try {
        res = await fetch(url, {
            redirect: "follow",
            signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
            headers: { "User-Agent": USER_AGENT, Accept: "text/html,application/xhtml+xml" },
        });
    }
    catch (err) {
        const message = err instanceof Error ? err.message : String(err);
        console.error(`[url-get] url=${url} error=fetch detail=${JSON.stringify(message)}`);
        return { ok: false, url, error: "fetch", message };
    }
    const httpStatus = res.status;
    if (!res.ok) {
        console.error(`[url-get] url=${url} error=http-${httpStatus}`);
        return { ok: false, url, error: "http", httpStatus, message: `HTTP ${httpStatus}` };
    }
    const contentType = (res.headers.get("content-type") ?? "").toLowerCase();
    if (contentType && !contentType.includes("html") && !contentType.includes("text/plain")) {
        console.error(`[url-get] url=${url} error=non-html contentType=${contentType}`);
        return {
            ok: false,
            url,
            error: "non-html",
            httpStatus,
            message: `unsupported content-type: ${contentType} (url-get handles server-rendered HTML only)`,
        };
    }
    const html = await res.text();
    const bytes = Buffer.byteLength(html, "utf-8");
    if (isChallenge(html)) {
        console.error(`[url-get] url=${url} error=challenge`);
        return {
            ok: false,
            url,
            error: "challenge",
            httpStatus,
            message: "page returned a bot-challenge interstitial, not content",
        };
    }
    const markdown = NodeHtmlMarkdown.translate(html).trim();
    const textLength = markdown.length;
    // HTTP 200 is success. Empty extractable text is a property of the response,
    // not an error: a JS-rendered SPA shell legitimately has zero markdown bytes
    // until the bundle runs. Surface `textLength` so the caller can decide
    // whether to fall back to browser-render; never invert the success signal.
    const accountDir = resolveAccountDir();
    if (!accountDir) {
        console.error(`[url-get] url=${url} error=no-account-dir`);
        return {
            ok: false,
            url,
            error: "no-account-dir",
            httpStatus,
            message: "cannot resolve account storage dir (ACCOUNT_DIR / PLATFORM_ROOT+ACCOUNT_ID unset)",
        };
    }
    const hash = createHash("sha256").update(url).digest("hex").slice(0, 32);
    const dir = join(accountDir, "url-get");
    const referencePath = join(dir, `${hash}.md`);
    const fetchedAt = new Date().toISOString();
    const fileBody = `<!-- url-get source=${url} fetchedAt=${fetchedAt} httpStatus=${httpStatus} -->\n\n${markdown}\n`;
    await mkdir(dir, { recursive: true });
    await writeFile(referencePath, fileBody, "utf-8");
    // The cleaned text is returned inline, capped — no model in the path. The
    // verbatim file on disk is the full ground truth; `text` is the same content
    // bounded so a large page does not flood the caller's context. An empty
    // `text` with textLength=0 is the JavaScript-rendered-shell signal; the
    // caller reads the numerics and retries with the browser MCP.
    const text = markdown.slice(0, TEXT_INPUT_CAP);
    console.error(`[url-get] url=${url} httpStatus=${httpStatus} bytes=${bytes} textLength=${textLength} ` +
        `refPath=${referencePath} mode=text (${Date.now() - t0}ms)`);
    return { ok: true, url, httpStatus, bytes, textLength, referencePath, text };
}
//# sourceMappingURL=url-get.js.map