import { matchesPatterns, canonicalForCrawl, canonicalForOutput, stripFragment } from "./url-utils.js";
import { RateLimiter } from "./rate-limiter.js";
import { RobotsParser } from "./robots.js";
import {
  parseSitemapIndex,
  parseSitemapEntries,
  sortSitemapEntries,
  extractSitemapUrlFromRobots
} from "./sitemap.js";
import { probeSitemap } from "./sitemap-first.js";
import { isIndexingEnabled, enqueueIndexCrawl } from "./index-to-vec.js";
import { getConfig } from "../config.js";
import { createLogger } from "../logger.js";
const log = createLogger("crawl");
class Crawler {
  fetchFn;
  rawFetchFn;
  rateLimiter = new RateLimiter();
  constructor(fetchFn, rawFetchFn) {
    this.fetchFn = fetchFn;
    this.rawFetchFn = rawFetchFn;
  }
  async crawl(input) {
    const strategy = input.strategy ?? "bfs";
    const maxDepth = input.max_depth ?? 2;
    const maxPages = input.max_pages ?? 20;
    const seedOrigin = new URL(input.url).origin;
    const config = getConfig();
    let robotsParser = null;
    if (config.respectRobotsTxt) {
      robotsParser = await this.fetchRobots(seedOrigin);
    }
    if (strategy === "auto") {
      const sitemapUrls = await probeSitemap(seedOrigin, this.rawFetchFn);
      if (sitemapUrls && sitemapUrls.length > 0) {
        log.info("auto strategy: using sitemap", { origin: seedOrigin, urls: sitemapUrls.length });
        return this.crawlFromExplicitUrls(input, sitemapUrls, maxPages, robotsParser);
      }
      log.info("auto strategy: no sitemap found, falling back to BFS", { origin: seedOrigin });
      return this.crawlTraversal(input, seedOrigin, maxDepth, maxPages, "bfs", robotsParser);
    }
    if (strategy === "sitemap") {
      return this.crawlSitemap(input, seedOrigin, maxPages, robotsParser);
    }
    const traversalStrategy = strategy === "map" ? "bfs" : strategy;
    return this.crawlTraversal(input, seedOrigin, maxDepth, maxPages, traversalStrategy, robotsParser);
  }
  robotsTxtContent = null;
  async fetchRobots(origin) {
    try {
      const result = await this.rawFetchFn(`${origin}/robots.txt`);
      if (result.statusCode === 200 && result.html) {
        this.robotsTxtContent = result.html;
        const parser = new RobotsParser(result.html);
        const crawlDelay = parser.getCrawlDelay();
        if (crawlDelay !== null) {
          const domain = new URL(origin).hostname;
          this.rateLimiter.setRobotsCrawlDelay(domain, crawlDelay);
        }
        return parser;
      }
    } catch {
      log.debug("Could not fetch robots.txt", { origin });
    }
    return null;
  }
  async crawlTraversal(input, seedOrigin, maxDepth, maxPages, strategy, robotsParser) {
    const visited = /* @__PURE__ */ new Set();
    const pages = [];
    const allLinks = [];
    const seenEdges = /* @__PURE__ */ new Set();
    const indexing = isIndexingEnabled();
    const queue = [[input.url, 0]];
    visited.add(canonicalForCrawl(input.url));
    while (queue.length > 0 && pages.length < maxPages) {
      const next = strategy === "dfs" ? queue.pop() : queue.shift();
      const [url, depth] = next;
      if (robotsParser && !robotsParser.isAllowed(new URL(url).pathname)) {
        log.debug("Blocked by robots.txt", { url });
        continue;
      }
      const release = await this.rateLimiter.acquire(url);
      let fetchResult;
      try {
        fetchResult = await this.fetchFn(url);
      } catch (err) {
        log.warn("Fetch failed during crawl", { url, error: String(err) });
        release();
        continue;
      }
      release();
      if (fetchResult.error) {
        log.warn("Fetch returned error", { url, error: fetchResult.error });
        continue;
      }
      const item = {
        url: canonicalForOutput(fetchResult.url),
        title: fetchResult.title,
        markdown: fetchResult.markdown,
        depth
      };
      pages.push(item);
      if (indexing) await enqueueIndexCrawl(item);
      if (depth < maxDepth) {
        const newLinks = this.filterLinks(fetchResult.links, seedOrigin, visited, input.include_patterns, input.exclude_patterns, robotsParser);
        for (const link of newLinks) {
          const canonical = canonicalForCrawl(link);
          if (visited.has(canonical)) continue;
          visited.add(canonical);
          queue.push([link, depth + 1]);
        }
        if (input.extract_links) {
          addUniqueEdges(allLinks, seenEdges, url, fetchResult.links);
        }
      }
    }
    return {
      pages,
      total_found: visited.size,
      crawled: pages.length,
      ...input.extract_links ? { links: allLinks } : {}
    };
  }
  filterLinks(links, seedOrigin, visited, includePatterns, excludePatterns, robotsParser) {
    const filtered = links.filter((link) => {
      try {
        const parsed = new URL(link);
        if (parsed.origin !== seedOrigin) return false;
        if (visited.has(canonicalForCrawl(link))) return false;
        if (!matchesPatterns(link, includePatterns, excludePatterns)) return false;
        if (robotsParser && !robotsParser.isAllowed(parsed.pathname)) return false;
        return true;
      } catch {
        return false;
      }
    });
    return filtered.sort((a, b) => {
      const aDoc = isDocPage(a) ? 0 : 1;
      const bDoc = isDocPage(b) ? 0 : 1;
      return aDoc - bDoc;
    });
  }
  async crawlSitemap(input, seedOrigin, maxPages, robotsParser) {
    const sitemapUrls = await this.discoverSitemapUrls(seedOrigin, this.robotsTxtContent);
    if (sitemapUrls.length === 0) {
      log.info("No sitemap found, falling back to BFS");
      return this.crawlTraversal(input, seedOrigin, input.max_depth ?? 2, maxPages, "bfs", robotsParser);
    }
    return this.crawlFromExplicitUrls(input, sitemapUrls, maxPages, robotsParser);
  }
  /**
   * Crawl an explicit list of URLs (e.g. from a sitemap probe). Applies
   * include/exclude patterns, robots.txt, max_pages, and rate limits the
   * same way as crawlSitemap.
   */
  async crawlFromExplicitUrls(input, urls, maxPages, robotsParser) {
    const seenCanonical = /* @__PURE__ */ new Set();
    const dedupedUrls = [];
    for (const url of urls) {
      const canonical = canonicalForCrawl(url);
      if (seenCanonical.has(canonical)) continue;
      seenCanonical.add(canonical);
      dedupedUrls.push(url);
    }
    const filtered = dedupedUrls.filter(
      (url) => matchesPatterns(url, input.include_patterns, input.exclude_patterns)
    );
    const totalFound = filtered.length;
    const toFetch = filtered.slice(0, maxPages);
    const pages = [];
    const allLinks = [];
    const seenEdges = /* @__PURE__ */ new Set();
    const indexing = isIndexingEnabled();
    for (const url of toFetch) {
      if (pages.length >= maxPages) break;
      if (robotsParser && !robotsParser.isAllowed(new URL(url).pathname)) continue;
      const release = await this.rateLimiter.acquire(url);
      try {
        const result = await this.fetchFn(url);
        release();
        if (!result.error) {
          const item = { url: canonicalForOutput(result.url), title: result.title, markdown: result.markdown, depth: 0 };
          pages.push(item);
          if (indexing) await enqueueIndexCrawl(item);
          if (input.extract_links) {
            addUniqueEdges(allLinks, seenEdges, url, result.links);
          }
        }
      } catch (err) {
        release();
        log.warn("Sitemap fetch failed", { url, error: String(err) });
      }
    }
    return {
      pages,
      total_found: totalFound,
      crawled: pages.length,
      ...input.extract_links ? { links: allLinks } : {}
    };
  }
  async discoverSitemapUrls(origin, robotsTxt) {
    const sitemapLocations = [];
    if (robotsTxt) {
      sitemapLocations.push(...extractSitemapUrlFromRobots(robotsTxt));
    }
    if (sitemapLocations.length === 0) {
      sitemapLocations.push(`${origin}/sitemap.xml`);
    }
    const allEntries = [];
    for (const sitemapUrl of sitemapLocations) {
      try {
        const result = await this.rawFetchFn(sitemapUrl);
        if (result.statusCode !== 200) continue;
        const indexUrls = parseSitemapIndex(result.html);
        if (indexUrls.length > 0) {
          for (const subUrl of indexUrls) {
            try {
              const subResult = await this.rawFetchFn(subUrl);
              if (subResult.statusCode === 200) {
                allEntries.push(...parseSitemapEntries(subResult.html));
              }
            } catch {
            }
          }
        } else {
          allEntries.push(...parseSitemapEntries(result.html));
        }
      } catch {
      }
    }
    return sortSitemapEntries(allEntries).map((e) => e.url);
  }
}
const DOC_PATH_PATTERNS = ["/docs/", "/guide/", "/api/", "/reference/"];
function isDocPage(url) {
  const path = new URL(url).pathname.toLowerCase();
  return DOC_PATH_PATTERNS.some((p) => path.includes(p));
}
function addUniqueEdges(edges, seen, from, links) {
  for (const link of links) {
    const canonicalTo = stripFragment(link);
    const key = `${from}\0${canonicalTo}`;
    if (seen.has(key)) continue;
    seen.add(key);
    edges.push({ from, to: canonicalTo });
  }
}
export {
  Crawler
};
//# sourceMappingURL=crawler.js.map