import { describeFetchError } from "../fetch/error-describe.js";
import { getExtractProvider } from "../providers/extract-provider.js";
import { getCachedContent, cacheContent, isCacheUsable } from "../cache/store.js";
import { getConfig } from "../config.js";
import { extractSection } from "../extraction/markdown.js";
import { detectChange } from "../cache/change-detector.js";
import { getEmbeddingService } from "../embedding/embed.js";
import { truncateSmartly, applyOutputBudget } from "../search/truncate.js";
import { buildEvidenceFromMarkdown } from "../search/evidence.js";
import { resolveMode } from "../util/mode.js";
import { createLogger } from "../logger.js";
const log = createLogger("fetch");
const DEFAULT_MAX_TOKENS_OUT = 4e3;
const DEFAULT_FETCH_BODY_TOKENS = 16e3;
const AUX_FIELD_CAP_WHEN_CHARS_BOUNDED = 50;
const AUX_FIELD_CAP_WHEN_TIGHT = 20;
function validateFetchUrl(raw) {
  if (typeof raw !== "string" || raw.length === 0) {
    return { ok: false, reason: "url is required and must be a non-empty string" };
  }
  const portMatch = raw.match(/^https?:\/\/(localhost|127\.0\.0\.1|\[?::1\]?)(?::([^\/?#]*))?/i);
  if (portMatch && portMatch[2] !== void 0) {
    const portStr = portMatch[2];
    const portNum = Number(portStr);
    if (!/^\d+$/.test(portStr) || !Number.isFinite(portNum) || portNum < 1 || portNum > 65535) {
      return {
        ok: false,
        reason: `invalid port "${portStr}" \u2014 localhost URLs require a valid port in 1-65535`,
        hint: "Use a port in 1-65535 (e.g. localhost:3000). Localhost itself is allowed for fetch/crawl; only the port is rejected."
      };
    }
  }
  if (!URL.canParse(raw)) {
    return {
      ok: false,
      reason: `url is not a valid absolute URL: ${JSON.stringify(raw)}`,
      hint: 'Pass a fully qualified http(s) URL (e.g. "https://example.com/path").'
    };
  }
  return { ok: true };
}
function capAuxFields(out, maxContentChars) {
  if (maxContentChars === void 0) return;
  const cap = maxContentChars <= 4e3 ? AUX_FIELD_CAP_WHEN_TIGHT : AUX_FIELD_CAP_WHEN_CHARS_BOUNDED;
  if (out.links && out.links.length > cap) out.links = out.links.slice(0, cap);
  if (out.images && out.images.length > cap) out.images = out.images.slice(0, cap);
}
async function attachEvidence(output, input, markdown) {
  if (!markdown) return;
  const includeFull = input.include_full_markdown ?? true;
  const maxTokensOut = input.max_tokens_out ?? DEFAULT_MAX_TOKENS_OUT;
  const evidence = await buildEvidenceFromMarkdown(
    output.title || output.url,
    output.title,
    output.url,
    markdown,
    { maxTokensOut, maxItems: 1 }
  );
  if (evidence.length > 0) output.evidence = evidence;
  if (!includeFull) {
    output.markdown = "";
  } else if (output.markdown) {
    output.markdown = applyOutputBudget(output.markdown, {
      maxTokensOut: input.max_tokens_out ?? DEFAULT_FETCH_BODY_TOKENS,
      maxChars: input.max_chars
    });
  }
}
function formatCachedResponse(cached, input) {
  let markdown = cached.markdown;
  let sectionMatched;
  if (input.section) {
    const result = extractSection(markdown, input.section, input.section_index);
    markdown = result.content;
    sectionMatched = result.matched;
  }
  if (input.max_chars && markdown.length > input.max_chars) {
    markdown = markdown.slice(0, input.max_chars);
  }
  if (input.max_content_chars !== void 0) {
    markdown = truncateSmartly(markdown, input.max_content_chars);
  }
  if (sectionMatched === false) {
    markdown = "";
  }
  const out = {
    url: cached.url,
    title: cached.title,
    markdown,
    metadata: {
      ...JSON.parse(cached.metadata || "{}"),
      ...sectionMatched !== void 0 ? { section_matched: sectionMatched } : {}
    },
    links: JSON.parse(cached.links || "[]"),
    images: JSON.parse(cached.images || "[]"),
    cached: true,
    cached_at: cached.fetchedAt,
    fetch_method: "cache",
    // Slice S1 (C2): surface the recorded HTTP status when available. Null
    // means the row predates the column; we simply omit the field.
    ...typeof cached.httpStatus === "number" ? { http_status: cached.httpStatus } : {}
  };
  capAuxFields(out, input.max_content_chars);
  return out;
}
async function handleFetch(input, router) {
  const mode = resolveMode(input.mode);
  const _fetchStart = Date.now();
  const stampTime = (out) => {
    out.response_time_ms = Date.now() - _fetchStart;
    return out;
  };
  const urlValidation = validateFetchUrl(input.url);
  if (!urlValidation.ok) {
    return {
      ok: false,
      error: "invalid_url",
      error_reason: urlValidation.reason,
      stage: "fetch",
      hint: urlValidation.hint
    };
  }
  try {
    if (!input.force_refresh) {
      const cached = getCachedContent(input.url);
      if (cached && (!input.actions || input.actions.length === 0)) {
        const staleMaxSeconds = mode === "cache" ? getConfig().fastStaleMaxHours * 3600 : 0;
        const { usable, stale } = isCacheUsable(cached, { staleMaxSeconds });
        if (usable) {
          log.info("Serving from cache", { url: input.url, stale });
          const out2 = formatCachedResponse(cached, input);
          if (stale) out2.stale = true;
          const fullMarkdown = out2.markdown;
          await attachEvidence(out2, input, fullMarkdown);
          return { ok: true, data: stampTime(out2) };
        }
      }
    }
    if (mode === "cache") {
      return {
        ok: false,
        error: "cache_miss",
        error_reason: `URL not in cache: ${input.url}`,
        stage: "fetch",
        hint: "Use mode:default to fetch live, or run search/crawl first to populate cache"
      };
    }
    const raw = await router.fetch(input.url, {
      renderJs: input.render_js ?? "auto",
      useAuth: input.use_auth ?? false,
      headers: input.headers,
      screenshot: input.screenshot,
      actions: input.actions,
      mode
    });
    if ("error" in raw && typeof raw.error === "string") {
      const stageErr = raw;
      return {
        ok: false,
        error: stageErr.error,
        error_reason: stageErr.error_reason ?? stageErr.error,
        stage: stageErr.stage ?? "fetch",
        ...stageErr.hint ? { hint: stageErr.hint } : {}
      };
    }
    const ct = raw.contentType?.toLowerCase() ?? "";
    const isMachineBody = !ct || /^(text\/plain|text\/markdown|application\/(json|xml|x-yaml))/i.test(ct);
    if (raw.statusCode >= 400 && isMachineBody) {
      const snippet = (raw.html ?? "").slice(0, 200).trim();
      return {
        ok: false,
        error: `http_${raw.statusCode}`,
        error_reason: `Upstream returned HTTP ${raw.statusCode}${snippet ? `: ${snippet}` : ""}`,
        stage: "fetch",
        hint: raw.statusCode === 404 ? "Check the URL \u2014 file/branch may have been removed or renamed" : "Retry later or check upstream status"
      };
    }
    const extractor = await getExtractProvider();
    const extraction = await extractor.extract(raw.html, raw.finalUrl, {
      maxChars: input.max_chars,
      section: input.section,
      sectionIndex: input.section_index,
      contentType: raw.contentType,
      pdfBuffer: raw.rawBuffer
    });
    let changeResult;
    try {
      changeResult = detectChange(raw.finalUrl, extraction.markdown, raw.statusCode);
    } catch (err) {
      log.warn("change detection failed", { url: raw.finalUrl, error: String(err) });
    }
    try {
      cacheContent(raw, extraction);
    } catch (err) {
      log.warn("failed to cache fetched content", { url: raw.finalUrl, error: String(err) });
    }
    try {
      const embeddingService = getEmbeddingService();
      if (embeddingService.isAvailable()) {
        embeddingService.embedAsync(raw.finalUrl, extraction.markdown);
      }
    } catch (err) {
      log.debug("embedding hook skipped", { error: String(err) });
    }
    let freshSectionMatched;
    if (input.section) {
      try {
        const probe = extractSection(extraction.markdown, input.section, input.section_index);
        if (probe && typeof probe.matched === "boolean") {
          freshSectionMatched = probe.matched;
        }
      } catch (err) {
        log.debug("section match probe failed", { url: raw.finalUrl, error: String(err) });
      }
    }
    let finalMarkdown = input.max_content_chars !== void 0 ? truncateSmartly(extraction.markdown, input.max_content_chars) : extraction.markdown;
    if (freshSectionMatched === false) {
      finalMarkdown = "";
    }
    const out = {
      url: raw.finalUrl,
      title: extraction.title,
      markdown: finalMarkdown,
      metadata: {
        ...extraction.metadata,
        ...freshSectionMatched !== void 0 ? { section_matched: freshSectionMatched } : {}
      },
      links: extraction.links,
      images: extraction.images,
      screenshot: raw.screenshot,
      cached: false,
      action_results: raw.actionResults,
      // Propagate the router-chosen tier name onto the public response so
      // callers can audit which path served the bytes (P2 visibility).
      fetch_method: raw.method,
      // Slice S1 (C2): always surface the upstream status code on fresh
      // fetches so callers / cache consumers can distinguish 200 / 404 /
      // 5xx pages that may extract to a usable HTML body.
      ...typeof raw.statusCode === "number" ? { http_status: raw.statusCode } : {},
      ...raw.jsRequired ? { js_required: true } : {},
      ...changeResult?.changed ? {
        changed: true,
        previous_hash: changeResult.previousHash,
        diff_summary: changeResult.diffSummary
      } : {},
      // Per-site structured JSON (e.g. Reddit `comments[]`, YouTube
      // `caption_tracks[]`, Amazon `asin`/`price`). Populated by the routed
      // extractor for sites with a site-specific extractor; absent otherwise.
      // Surfacing at top level (rather than nesting under `extra`) matches
      // the existing house style for `evidence` / `screenshot`.
      ...extraction.site_data ? { site_data: extraction.site_data } : {},
      // Slice S7 (C5): partial-success marker. When a Reddit / Amazon site
      // extractor detected an anti-bot or page-not-found body, the routed
      // extractor sets `site_data_blocked` and we surface it on the envelope
      // as `fetch_failed` so callers branch honestly. site_data is
      // intentionally absent in that case.
      ...extraction.site_data_blocked ? { fetch_failed: extraction.site_data_blocked } : {}
    };
    capAuxFields(out, input.max_content_chars);
    await attachEvidence(out, input, finalMarkdown);
    return { ok: true, data: stampTime(out) };
  } catch (err) {
    log.error("Fetch failed", { url: input.url, error: String(err) });
    const described = describeFetchError(err);
    return {
      ok: false,
      error: "fetch_failed",
      error_reason: described.reason,
      stage: "fetch",
      ...described.hint ? { hint: described.hint } : {}
    };
  }
}
export {
  handleFetch
};
//# sourceMappingURL=fetch.js.map