import { getExtractProvider } from "../providers/extract-provider.js";
import { cacheContent } from "../cache/store.js";
import { getEmbeddingService } from "../embedding/embed.js";
import { truncateSmartly } from "./truncate.js";
import { createLogger } from "../logger.js";
const log = createLogger("search");
async function fetchOne(url, router, ctx) {
  if (Date.now() >= ctx.totalDeadline) {
    return { error: "total_timeout" };
  }
  try {
    const raw = await Promise.race([
      router.fetch(url, {
        renderJs: "auto",
        ...ctx.forceRefresh && { force_refresh: true }
      }),
      new Promise(
        (_, reject) => setTimeout(() => reject(new Error("timeout")), ctx.fetchTimeoutMs)
      )
    ]);
    const extractor = await getExtractProvider();
    const extraction = await extractor.extract(raw.html, raw.finalUrl, {
      maxChars: ctx.contentMaxChars,
      contentType: raw.contentType
    });
    try {
      cacheContent(raw, extraction);
    } catch (err) {
      log.warn("failed to cache search result", { url, error: String(err) });
    }
    try {
      const embeddingService = getEmbeddingService();
      if (embeddingService.isAvailable()) {
        embeddingService.embedAsync(raw.finalUrl, extraction.markdown);
      }
    } catch (err) {
      log.debug("embedding hook skipped for search result", { error: String(err) });
    }
    const md = ctx.maxContentChars !== void 0 ? truncateSmartly(extraction.markdown, ctx.maxContentChars) : extraction.markdown;
    return { content: md };
  } catch (err) {
    const msg = err instanceof Error ? err.message : String(err);
    log.debug("content fetch failed", { url, error: msg });
    return { error: msg };
  }
}
async function fetchContentForResults(results, router, ctx) {
  const cap = ctx.maxFetches !== void 0 ? ctx.maxFetches : results.length;
  const fetchTargets = results.slice(0, cap);
  const attempted = new Set(fetchTargets.map((r) => r.url));
  const fetched = await Promise.all(
    fetchTargets.map((r) => fetchOne(r.url, router, ctx))
  );
  if (cap > 1 && results.length > cap) {
    const originalFailedCount = fetched.filter((f) => f.content === void 0).length;
    let backupsAccepted = 0;
    let nextBackupIdx = cap;
    while (Date.now() < ctx.totalDeadline && nextBackupIdx < results.length && backupsAccepted < originalFailedCount) {
      const stillNeeded = originalFailedCount - backupsAccepted;
      const wave = [];
      while (wave.length < stillNeeded && nextBackupIdx < results.length) {
        const candidate = results[nextBackupIdx];
        nextBackupIdx++;
        if (attempted.has(candidate.url)) continue;
        attempted.add(candidate.url);
        wave.push(candidate);
      }
      if (wave.length === 0) break;
      const waveResults = await Promise.all(
        wave.map((r) => fetchOne(r.url, router, ctx))
      );
      for (let i = 0; i < wave.length; i++) {
        if (waveResults[i].content === void 0) continue;
        if (backupsAccepted >= originalFailedCount) break;
        fetchTargets.push(wave[i]);
        fetched.push(waveResults[i]);
        backupsAccepted++;
      }
    }
  }
  let totalCharsUsed = 0;
  for (let i = 0; i < fetchTargets.length; i++) {
    const result = fetchTargets[i];
    const { content, error } = fetched[i];
    if (error) {
      result.fetch_failed = error;
      continue;
    }
    if (content === void 0) continue;
    if (totalCharsUsed >= ctx.maxTotalChars) {
      result.content_truncated = true;
      continue;
    }
    let out = content;
    const remaining = ctx.maxTotalChars - totalCharsUsed;
    if (out.length > remaining) {
      out = out.slice(0, remaining);
      result.content_truncated = true;
    }
    totalCharsUsed += out.length;
    result.markdown_content = out;
  }
}
export {
  fetchContentForResults
};
//# sourceMappingURL=content-fetch.js.map