import { parseHTML } from "linkedom";
import { matchesPatterns, canonicalForOutput } from "./url-utils.js";
import { parseSitemap, parseSitemapIndex, extractSitemapUrlFromRobots } from "./sitemap.js";
import { createLogger } from "../logger.js";
const log = createLogger("crawl");
const IGNORED_PROTOCOLS = ["javascript:", "mailto:", "tel:", "data:", "blob:", "ftp:"];
function extractLinks(html, origin) {
  if (!html || !html.trim()) return [];
  try {
    const { document: doc } = parseHTML(html);
    const anchors = doc.querySelectorAll("a[href]");
    const seen = /* @__PURE__ */ new Set();
    const links = [];
    const parsedOrigin = new URL(origin).origin;
    for (const anchor of Array.from(anchors)) {
      const href = anchor.getAttribute("href");
      if (!href) continue;
      const trimmed = href.trim();
      if (!trimmed) continue;
      if (trimmed.startsWith("#")) continue;
      if (IGNORED_PROTOCOLS.some((p) => trimmed.toLowerCase().startsWith(p))) continue;
      try {
        const resolved = new URL(trimmed, origin);
        if (resolved.origin !== parsedOrigin) continue;
        resolved.hash = "";
        const normalized = resolved.href;
        if (!seen.has(normalized)) {
          seen.add(normalized);
          links.push(normalized);
        }
      } catch {
        log.debug("Failed to resolve URL", { href: trimmed, origin });
      }
    }
    return links;
  } catch (err) {
    log.debug("Failed to parse HTML for link extraction", { error: String(err) });
    return [];
  }
}
async function mapUrls(input, fetchFn) {
  const maxDepth = input.max_depth ?? 3;
  const maxPages = input.max_pages ?? 200;
  let origin;
  try {
    origin = new URL(input.url).origin;
  } catch (err) {
    return {
      urls: [],
      total_found: 0,
      sitemap_found: false,
      error: `Invalid seed URL: ${String(err)}`
    };
  }
  const seedCanonical = canonicalForOutput(input.url);
  const discovered = /* @__PURE__ */ new Set([seedCanonical]);
  const queued = /* @__PURE__ */ new Set([seedCanonical]);
  const queue = [{ url: input.url, depth: 0 }];
  let sitemapFound = false;
  try {
    const sitemapUrls = await discoverSitemapUrls(origin, fetchFn);
    if (sitemapUrls.length > 0) {
      sitemapFound = true;
      for (const sitemapUrl of sitemapUrls) {
        if (discovered.size >= maxPages) break;
        if (matchesPatterns(sitemapUrl, input.include_patterns, input.exclude_patterns)) {
          const canonical = canonicalForOutput(sitemapUrl);
          if (discovered.has(canonical)) continue;
          discovered.add(canonical);
          if (!queued.has(canonical)) {
            queued.add(canonical);
            queue.push({ url: sitemapUrl, depth: 0 });
          }
        }
      }
    }
  } catch (err) {
    log.debug("Sitemap discovery failed, continuing with BFS only", { error: String(err) });
  }
  let seedError;
  while (queue.length > 0 && discovered.size < maxPages) {
    const current = queue.shift();
    if (current.depth > maxDepth) continue;
    try {
      const { html } = await fetchFn(current.url);
      const links = extractLinks(html, origin);
      for (const link of links) {
        if (discovered.size >= maxPages) break;
        const canonical = canonicalForOutput(link);
        if (discovered.has(canonical)) continue;
        if (!matchesPatterns(link, input.include_patterns, input.exclude_patterns)) continue;
        discovered.add(canonical);
        if (current.depth + 1 <= maxDepth && !queued.has(canonical)) {
          queued.add(canonical);
          queue.push({ url: link, depth: current.depth + 1 });
        }
      }
    } catch (err) {
      if (current.url === input.url && current.depth === 0) {
        seedError = String(err);
        log.warn("Seed URL fetch failed during map", { url: current.url, error: String(err) });
      } else {
        log.debug("Child page fetch failed during map, skipping", { url: current.url, error: String(err) });
      }
    }
  }
  const urls = Array.from(discovered);
  return {
    urls,
    total_found: urls.length,
    sitemap_found: sitemapFound,
    ...seedError !== void 0 ? { error: seedError } : {}
  };
}
async function discoverSitemapUrls(origin, fetchFn) {
  const sitemapLocations = [];
  try {
    const { html: robotsTxt } = await fetchFn(`${origin}/robots.txt`);
    const fromRobots = extractSitemapUrlFromRobots(robotsTxt);
    sitemapLocations.push(...fromRobots);
  } catch {
    log.debug("robots.txt not found, trying default sitemap location");
  }
  if (sitemapLocations.length === 0) {
    sitemapLocations.push(`${origin}/sitemap.xml`);
  }
  const allUrls = [];
  for (const sitemapUrl of sitemapLocations) {
    try {
      const { html: sitemapXml } = await fetchFn(sitemapUrl);
      const indexUrls = parseSitemapIndex(sitemapXml);
      if (indexUrls.length > 0) {
        for (const subUrl of indexUrls) {
          try {
            const { html: subXml } = await fetchFn(subUrl);
            allUrls.push(...parseSitemap(subXml));
          } catch {
            log.debug("Failed to fetch sub-sitemap", { url: subUrl });
          }
        }
      } else {
        allUrls.push(...parseSitemap(sitemapXml));
      }
    } catch {
      log.debug("Sitemap fetch failed", { url: sitemapUrl });
    }
  }
  return allUrls;
}
export {
  extractLinks,
  mapUrls
};
//# sourceMappingURL=mapper.js.map