import { Crawler } from "../crawl/crawler.js";
import { deduplicatePages } from "../crawl/dedup.js";
import { mapUrls } from "../crawl/mapper.js";
import { handleFetch } from "./fetch.js";
import {
  buildEvidenceFromMarkdown,
  applyAggregateMarkdownBudget
} from "../search/evidence.js";
import { countTokens } from "../search/tokens.js";
import { createLogger } from "../logger.js";
const log = createLogger("crawl");
const DEFAULT_MAX_TOTAL_CHARS = 1e5;
const DEFAULT_MAX_TOKENS_OUT = 4e3;
async function handleCrawl(input, router) {
  const _start = Date.now();
  try {
    if (input.strategy === "map") {
      return handleMapStrategy(input, router);
    }
    const fetchFn = async (url) => {
      const r = await handleFetch({ url, use_auth: input.use_auth, include_full_markdown: true }, router);
      if (!r.ok) {
        return {
          url,
          title: "",
          markdown: "",
          metadata: {},
          links: [],
          images: [],
          cached: false,
          error: r.error_reason
        };
      }
      return r.data;
    };
    const rawFetchFn = async (url) => router.fetch(url, { renderJs: "never" });
    const crawler = new Crawler(fetchFn, rawFetchFn);
    const result = await crawler.crawl(input);
    const domain = new URL(input.url).hostname;
    const dedupedPages = deduplicatePages(
      result.pages.map((p) => ({ url: p.url, markdown: p.markdown })),
      domain
    );
    const pages = result.pages.map((page, i) => ({
      ...page,
      markdown: dedupedPages[i]?.markdown ?? page.markdown
    }));
    const maxTotalChars = input.max_total_chars ?? DEFAULT_MAX_TOTAL_CHARS;
    const budgetedPages = [];
    let charCount = 0;
    for (const page of pages) {
      if (charCount + page.markdown.length > maxTotalChars && budgetedPages.length > 0) {
        break;
      }
      budgetedPages.push(page);
      charCount += page.markdown.length;
    }
    const droppedOverBudget = result.crawled - budgetedPages.length;
    log.info("Crawl complete", {
      url: input.url,
      crawled: result.crawled,
      returned: budgetedPages.length,
      totalChars: charCount,
      droppedOverBudget
    });
    const out = {
      pages: budgetedPages,
      total_found: result.total_found,
      crawled: budgetedPages.length,
      ...droppedOverBudget > 0 ? { dropped_over_budget: droppedOverBudget } : {},
      ...result.links ? { links: result.links } : {}
    };
    await attachEvidence(out, input);
    out.response_time_ms = Date.now() - _start;
    return out;
  } catch (err) {
    log.error("Crawl failed", { url: input.url, error: String(err) });
    return {
      pages: [],
      total_found: 0,
      crawled: 0,
      response_time_ms: Date.now() - _start,
      error: err instanceof Error ? err.message : String(err)
    };
  }
}
function buildPageExcerpt(markdown, maxChars = 600) {
  if (!markdown) return "";
  const paragraphs = markdown.split(/\n\n+/).map((p) => p.trim()).filter(Boolean);
  let out = "";
  for (const p of paragraphs) {
    if (out.length + p.length + 2 > maxChars) {
      const remaining = maxChars - out.length;
      if (remaining > 80) out += (out ? "\n\n" : "") + p.slice(0, remaining) + "\u2026";
      break;
    }
    out += (out ? "\n\n" : "") + p;
  }
  return out;
}
async function attachEvidence(out, input) {
  if (out.pages.length === 0) return;
  const includeFull = input.include_full_markdown ?? true;
  const maxTokensOut = input.max_tokens_out ?? DEFAULT_MAX_TOKENS_OUT;
  let used = 0;
  for (const page of out.pages) {
    if (!page.markdown) continue;
    const remaining = maxTokensOut - used;
    if (remaining <= 0) break;
    const evs = await buildEvidenceFromMarkdown(
      page.title || page.url,
      page.title,
      page.url,
      page.markdown,
      { maxItems: 1, maxTokensOut: remaining }
    );
    if (evs.length > 0) {
      page.evidence = evs;
      for (const ev of evs) used += countTokens(ev.excerpt);
    }
  }
  if (!includeFull) {
    for (const page of out.pages) {
      if (!page.evidence || page.evidence.length === 0) {
        const excerpt = buildPageExcerpt(page.markdown);
        if (excerpt) page.excerpt = excerpt;
      }
      page.markdown = "";
    }
  } else {
    applyAggregateMarkdownBudget(
      out.pages,
      (p) => p.markdown ?? "",
      (p, body) => {
        p.markdown = body;
      },
      { maxTokensOut }
    );
  }
}
async function handleMapStrategy(input, router) {
  const httpFetchFn = async (url) => {
    const raw = await router.fetch(url, { renderJs: "never" });
    return { html: raw.html, finalUrl: raw.finalUrl, statusCode: raw.statusCode };
  };
  try {
    const mapResult = await mapUrls(
      {
        url: input.url,
        max_depth: input.max_depth,
        max_pages: input.max_pages,
        include_patterns: input.include_patterns,
        exclude_patterns: input.exclude_patterns
      },
      httpFetchFn
    );
    log.info("Map complete", {
      url: input.url,
      total_found: mapResult.total_found,
      sitemap_found: mapResult.sitemap_found
    });
    return {
      urls: mapResult.urls,
      total_found: mapResult.total_found,
      sitemap_found: mapResult.sitemap_found,
      crawled: 0,
      ...mapResult.error ? { error: mapResult.error } : {}
    };
  } catch (err) {
    log.error("Map strategy failed", { url: input.url, error: String(err) });
    return {
      urls: [],
      total_found: 0,
      sitemap_found: false,
      crawled: 0,
      error: err instanceof Error ? err.message : String(err)
    };
  }
}
export {
  handleCrawl
};
//# sourceMappingURL=crawl.js.map