import { extractKeyTerms, buildFTS5Query } from "../embedding/key-terms.js";
import { reciprocalRankFusion, sortByRRFScore } from "./rrf.js";
import { searchCache, getCachedContent, normalizeUrl, getCacheStats } from "../cache/store.js";
import { filterByDomains } from "./filters.js";
import { handleSearch } from "../tools/search.js";
import { getExtractProvider } from "../providers/extract-provider.js";
import { getEmbeddingService } from "../embedding/embed.js";
import { createLogger } from "../logger.js";
import { getConfig } from "../config.js";
import { selectMode } from "./find-similar/mode.js";
import { crawlRank } from "./find-similar/crawl-rank.js";
const log = createLogger("search");
const DEFAULT_MAX_RESULTS = 10;
const MAX_FTS5_CANDIDATES = 20;
const MAX_EMBEDDING_CANDIDATES = 20;
const WEB_SEARCH_QUERY_COUNT = 3;
async function findSimilar(input, engines, router, backendStatus) {
  const start = Date.now();
  const embeddingAvailable = checkEmbeddingAvailable();
  const initialCacheSize = safeCacheCount();
  const initialEmbedIndexSize = safeEmbedIndexSize();
  try {
    const mode = selectMode(input);
    if (mode === "crawl-rank") {
      const seed = input.url?.trim();
      if (!seed) {
        return {
          results: [],
          method: "fts5",
          cache_hits: 0,
          search_hits: 0,
          embedding_available: embeddingAvailable,
          error: "crawl-rank mode requires a url",
          total_time_ms: Date.now() - start
        };
      }
      const cr = await crawlRank(seed, input, router);
      return { ...cr, total_time_ms: Date.now() - start };
    }
    const url = input.url?.trim();
    const concept = input.concept?.trim();
    if (!url && !concept) {
      return {
        results: [],
        method: "fts5",
        cache_hits: 0,
        search_hits: 0,
        embedding_available: embeddingAvailable,
        error: "Either url or concept must be provided",
        total_time_ms: Date.now() - start
      };
    }
    const maxResults = input.max_results ?? DEFAULT_MAX_RESULTS;
    const includeCache = input.include_cache ?? true;
    const includeWeb = input.include_web ?? true;
    const signal = await prepareSignal(url, concept, router);
    if (signal.terms.length === 0 && !signal.queryText) {
      log.warn("no key terms or query text extracted, falling back to web search");
      if (!includeWeb) {
        return {
          results: [],
          method: "fts5",
          cache_hits: 0,
          search_hits: 0,
          embedding_available: embeddingAvailable,
          error: "Could not extract key terms from input and web search is disabled",
          total_time_ms: Date.now() - start
        };
      }
    }
    let cacheResults = [];
    const fts5RankMap = /* @__PURE__ */ new Map();
    let embeddingResults = [];
    const embeddingRankMap = /* @__PURE__ */ new Map();
    await Promise.all([
      (async () => {
        if (includeCache && signal.terms.length > 0) {
          cacheResults = runFTS5Search(
            signal.terms,
            signal.inputNormalizedUrl,
            input.include_domains,
            input.exclude_domains,
            MAX_FTS5_CANDIDATES,
            fts5RankMap
          );
          log.debug("FTS5 search complete", { hits: cacheResults.length });
        }
      })(),
      (async () => {
        if (includeCache && embeddingAvailable && signal.queryText) {
          embeddingResults = await runEmbeddingSearch(
            signal.queryText,
            signal.inputNormalizedUrl,
            input.include_domains,
            input.exclude_domains,
            MAX_EMBEDDING_CANDIDATES,
            embeddingRankMap
          );
          log.debug("embedding search complete", { hits: embeddingResults.length });
        }
      })()
    ]);
    let searchResults = [];
    const searchRankMap = /* @__PURE__ */ new Map();
    const combinedLocalHits = /* @__PURE__ */ new Set();
    for (const r of cacheResults) combinedLocalHits.add(safeNormalize(r.url));
    for (const r of embeddingResults) combinedLocalHits.add(safeNormalize(r.url));
    if (combinedLocalHits.size < maxResults && includeWeb) {
      searchResults = await runWebSearchFallback(
        signal,
        engines,
        router,
        backendStatus,
        maxResults,
        signal.inputNormalizedUrl,
        input.include_domains,
        input.exclude_domains,
        searchRankMap
      );
      log.debug("web search fallback complete", { hits: searchResults.length });
      if (embeddingAvailable && signal.queryText && searchResults.length > 0) {
        const freshEmbeddingResults = await runEmbeddingSearch(
          signal.queryText,
          signal.inputNormalizedUrl,
          input.include_domains,
          input.exclude_domains,
          MAX_EMBEDDING_CANDIDATES,
          embeddingRankMap
        );
        if (freshEmbeddingResults.length > 0) {
          embeddingResults = freshEmbeddingResults;
          log.debug("re-ran embedding search after web fallback", { hits: embeddingResults.length });
        }
      }
    }
    const rankedLists = [];
    if (fts5RankMap.size > 0) rankedLists.push(fts5RankMap);
    if (embeddingRankMap.size > 0) rankedLists.push(embeddingRankMap);
    if (searchRankMap.size > 0) rankedLists.push(searchRankMap);
    const allResults = mergeResults(cacheResults, embeddingResults, searchResults);
    let finalResults;
    let topRawScore = 0;
    if (rankedLists.length >= 1) {
      const fused = fuseResults(rankedLists, allResults, maxResults);
      finalResults = fused.results;
      topRawScore = fused.topRawScore;
    } else {
      finalResults = allResults.sort((a, b) => b.relevance_score - a.relevance_score).slice(0, maxResults);
    }
    const threshold = input.threshold ?? 0;
    if (threshold > 0) {
      finalResults = finalResults.filter(
        (r) => r.match_signals.fused_score >= threshold
      );
    }
    const method = determineMethod(
      cacheResults.length > 0,
      embeddingResults.length > 0,
      searchResults.length > 0
    );
    const cacheHits = finalResults.filter((r) => r.source === "cache").length;
    const searchHits = finalResults.filter((r) => r.source === "search").length;
    const coldStartThreshold = safeColdStartThreshold();
    const weakSignal = rankedLists.length >= 1 && finalResults.length > 0 && topRawScore > 0 && coldStartThreshold > 0 && topRawScore < coldStartThreshold;
    if (weakSignal) {
      for (const r of finalResults) {
        r.relevance_score = r.match_signals.fused_score;
      }
    }
    if (input.include_ranking_debug) {
      for (const r of finalResults) {
        const key = safeNormalize(r.url);
        const debug = {
          rrf_score: r.match_signals.fused_score
        };
        const fts = fts5RankMap.get(key);
        if (fts !== void 0) debug.fts5_rank = fts;
        const emb = embeddingRankMap.get(key);
        if (emb !== void 0) debug.embedding_rank = emb;
        const web = searchRankMap.get(key);
        if (web !== void 0) debug.web_rank = web;
        r.ranking_debug = debug;
      }
    }
    const queryForNote = (input.concept?.trim() || input.url?.trim() || "").slice(0, 200);
    const conceptMode = !!input.concept?.trim() && !input.url?.trim();
    const baseNote = buildColdStartNote(
      cacheHits,
      searchHits,
      embeddingAvailable,
      initialCacheSize,
      initialEmbedIndexSize,
      conceptMode,
      finalResults.length
    );
    const weakNote = weakSignal ? buildWeakSignalNote(queryForNote, topRawScore, coldStartThreshold) : void 0;
    const coldStart = [baseNote, weakNote].filter(Boolean).join(" ") || void 0;
    return {
      results: finalResults,
      method,
      cache_hits: cacheHits,
      search_hits: searchHits,
      embedding_available: embeddingAvailable,
      ...coldStart ? { cold_start: coldStart } : {},
      total_time_ms: Date.now() - start
    };
  } catch (err) {
    log.error("findSimilar failed", { error: String(err) });
    return {
      results: [],
      method: "fts5",
      cache_hits: 0,
      search_hits: 0,
      embedding_available: embeddingAvailable,
      error: `find_similar failed: ${err instanceof Error ? err.message : String(err)}`,
      total_time_ms: Date.now() - start
    };
  }
}
function checkEmbeddingAvailable() {
  try {
    const svc = getEmbeddingService();
    return svc.isAvailable() && svc.isSubprocessReady();
  } catch {
    return false;
  }
}
function safeCacheCount() {
  try {
    return getCacheStats().total_urls;
  } catch {
    return 0;
  }
}
function safeEmbedIndexSize() {
  try {
    return getEmbeddingService().getIndex().size();
  } catch {
    return 0;
  }
}
function buildColdStartNote(cacheHits, searchHits, embeddingAvailable, initialCacheSize, initialEmbedIndexSize, conceptMode, finalResultCount) {
  if (initialCacheSize === 0) {
    return "Cache is empty. Results come from live web search only. Use wigolo_fetch / wigolo_crawl to warm the cache, then re-run find_similar for hybrid local+web ranking.";
  }
  if (cacheHits === 0 && searchHits > 0) {
    return `No cache matches for this query (cache has ${initialCacheSize} pages overall). Results come from live web search. Use wigolo_fetch on relevant sources before re-running for hybrid ranking.`;
  }
  if (conceptMode && searchHits === 0 && finalResultCount > 0 && finalResultCount <= 2 && initialCacheSize >= 3) {
    return `Only ${finalResultCount} cache match${finalResultCount === 1 ? "" : "es"} for this concept query (cache has ${initialCacheSize} pages overall). The result is a thin local-cache signal and may not be representative. Enable include_web=true or run wigolo_crawl on relevant sources to corroborate.`;
  }
  if (!embeddingAvailable && initialCacheSize > 0) {
    return "Embeddings unavailable or index empty (cached pages have not been embedded yet). Falling back to FTS5 keyword ranking. Set up sentence-transformers to enable semantic matching.";
  }
  if (cacheHits === 0 && initialCacheSize < 20) {
    return `Cache has only ${initialCacheSize} pages. Add more context by fetching or crawling relevant sites before relying on find_similar for cross-source similarity.`;
  }
  if (!embeddingAvailable && initialEmbedIndexSize === 0) {
    return "Embedding index is empty. Semantic matching disabled until background embedding jobs catch up.";
  }
  return void 0;
}
function safeColdStartThreshold() {
  try {
    return getConfig().findSimilarColdStartThreshold;
  } catch {
    return 0.02;
  }
}
function buildWeakSignalNote(query, topRawScore, threshold) {
  const score = topRawScore.toFixed(4);
  const t = threshold.toFixed(4);
  const queryPart = query ? `Query "${query}": ` : "";
  return `${queryPart}top result is a weak match (raw signal ${score} below threshold ${t}). Per-result relevance_score replaced with raw fused_score so callers see actual signal strength. Tune WIGOLO_FIND_SIMILAR_COLD_START_THRESHOLD to adjust, or set it to 0 to disable.`;
}
function safeNormalize(url) {
  try {
    return normalizeUrl(url);
  } catch {
    return url;
  }
}
function mergeResults(...lists) {
  const seen = /* @__PURE__ */ new Map();
  for (const list of lists) {
    for (const r of list) {
      const key = safeNormalize(r.url);
      const existing = seen.get(key);
      if (!existing) {
        seen.set(key, r);
      } else {
        existing.match_signals = {
          ...existing.match_signals,
          ...r.match_signals,
          fused_score: existing.match_signals.fused_score
        };
      }
    }
  }
  return [...seen.values()];
}
async function prepareSignal(url, concept, router) {
  if (url) {
    return await prepareSignalFromUrl(url, router);
  }
  if (concept) {
    const terms = extractKeyTerms(concept, "");
    return { terms, title: concept, queryText: concept };
  }
  return { terms: [], title: "" };
}
async function prepareSignalFromUrl(url, router) {
  let normalizedInputUrl;
  try {
    normalizedInputUrl = normalizeUrl(url);
  } catch {
    normalizedInputUrl = url;
  }
  const cached = getCachedContent(url);
  if (cached) {
    const terms = extractKeyTerms(cached.markdown, cached.title);
    return {
      terms,
      title: cached.title,
      inputUrl: url,
      inputNormalizedUrl: normalizedInputUrl,
      queryText: cached.markdown
    };
  }
  try {
    log.info("fetching URL for signal extraction", { url });
    const raw = await router.fetch(url, { renderJs: "auto" });
    const extractor = await getExtractProvider();
    const extraction = await extractor.extract(raw.html, raw.finalUrl, {
      contentType: raw.contentType
    });
    const terms = extractKeyTerms(extraction.markdown, extraction.title);
    return {
      terms,
      title: extraction.title,
      inputUrl: url,
      inputNormalizedUrl: normalizedInputUrl,
      queryText: extraction.markdown
    };
  } catch (err) {
    log.warn("failed to fetch URL for signal extraction", { url, error: String(err) });
    const urlTerms = extractKeyTerms("", url);
    return {
      terms: urlTerms,
      title: url,
      inputUrl: url,
      inputNormalizedUrl: normalizedInputUrl
    };
  }
}
async function runEmbeddingSearch(queryText, excludeNormalizedUrl, includeDomains, excludeDomains, topK, rankMap) {
  try {
    const service = getEmbeddingService();
    if (!service.isAvailable() || !service.isSubprocessReady()) return [];
    if (service.getIndex().size() === 0) return [];
    const excludeUrls = excludeNormalizedUrl ? /* @__PURE__ */ new Set([excludeNormalizedUrl]) : void 0;
    const similar = await service.findSimilar(queryText, topK, excludeUrls);
    if (similar.length === 0) return [];
    const hydrated = [];
    for (const { url: nUrl, score } of similar) {
      const cached = getCachedContent(nUrl);
      hydrated.push({ entry: cached, url: nUrl, score });
    }
    const filterableInputs = hydrated.map((h) => ({
      url: h.entry?.url ?? h.url
    }));
    const filtered = filterByDomains(filterableInputs, includeDomains, excludeDomains);
    const allowedUrls = new Set(filtered.map((f) => f.url));
    const results = [];
    let rank = 0;
    for (const h of hydrated) {
      const displayUrl = h.entry?.url ?? h.url;
      if (!allowedUrls.has(displayUrl)) continue;
      rank++;
      rankMap.set(safeNormalize(displayUrl), rank);
      results.push({
        url: displayUrl,
        title: h.entry?.title ?? displayUrl,
        markdown: (h.entry?.markdown ?? "").slice(0, 5e3),
        relevance_score: h.score,
        source: "cache",
        match_signals: {
          embedding_rank: rank,
          fused_score: 0
        }
      });
    }
    return results;
  } catch (err) {
    log.warn("embedding search failed", { error: String(err) });
    return [];
  }
}
function runFTS5Search(terms, excludeNormalizedUrl, includeDomains, excludeDomains, maxCandidates, rankMap) {
  try {
    const fts5Query = buildFTS5Query(terms);
    if (!fts5Query) return [];
    let cached = searchCache(fts5Query);
    if (excludeNormalizedUrl) {
      cached = cached.filter((c) => {
        try {
          return normalizeUrl(c.url) !== excludeNormalizedUrl;
        } catch {
          return c.url !== excludeNormalizedUrl;
        }
      });
    }
    cached = filterByDomains(cached, includeDomains, excludeDomains);
    cached = cached.slice(0, maxCandidates);
    const results = [];
    for (let i = 0; i < cached.length; i++) {
      const entry = cached[i];
      let nUrl;
      try {
        nUrl = normalizeUrl(entry.url);
      } catch {
        nUrl = entry.url;
      }
      rankMap.set(nUrl, i + 1);
      results.push({
        url: entry.url,
        title: entry.title,
        markdown: entry.markdown.slice(0, 5e3),
        relevance_score: 0,
        source: "cache",
        match_signals: {
          fts5_rank: i + 1,
          fused_score: 0
        }
      });
    }
    return results;
  } catch (err) {
    log.error("FTS5 search failed", { error: String(err) });
    return [];
  }
}
async function runWebSearchFallback(signal, engines, router, backendStatus, maxResults, excludeNormalizedUrl, includeDomains, excludeDomains, rankMap) {
  try {
    const queries = generateSearchQueries(signal.terms, signal.title);
    if (queries.length === 0) return [];
    const allResults = [];
    const seenUrls = /* @__PURE__ */ new Set();
    if (excludeNormalizedUrl) {
      seenUrls.add(excludeNormalizedUrl);
    }
    for (const query of queries) {
      try {
        const searchResult = await handleSearch(
          {
            query,
            max_results: maxResults,
            include_content: true,
            include_domains: includeDomains,
            exclude_domains: excludeDomains
          },
          engines,
          router,
          backendStatus
        );
        if (!searchResult.ok) {
          log.warn("web search query failed", { query, error: searchResult.error_reason });
          continue;
        }
        const searchOutput = searchResult.data;
        for (const item of searchOutput.results) {
          let nUrl;
          try {
            nUrl = normalizeUrl(item.url);
          } catch {
            nUrl = item.url;
          }
          if (seenUrls.has(nUrl)) continue;
          seenUrls.add(nUrl);
          const rank = allResults.length + 1;
          rankMap.set(nUrl, rank);
          allResults.push({
            url: item.url,
            title: item.title,
            markdown: (item.markdown_content ?? item.snippet).slice(0, 5e3),
            relevance_score: item.relevance_score,
            source: "search",
            match_signals: {
              fused_score: 0
            }
          });
        }
      } catch (err) {
        log.warn("web search query failed", { query, error: String(err) });
      }
    }
    try {
      const embeddingService = getEmbeddingService();
      if (embeddingService.isAvailable() && embeddingService.isSubprocessReady()) {
        const embedPromises = allResults.filter((r) => r.markdown).slice(0, 10).map((r) => embeddingService.embedAndStore(r.url, r.markdown));
        await Promise.allSettled(embedPromises);
        log.debug("embedded web fallback results", { count: embedPromises.length });
      }
    } catch (err) {
      log.debug("embedding hook skipped for find_similar results", { error: String(err) });
    }
    return allResults;
  } catch (err) {
    log.error("web search fallback failed", { error: String(err) });
    return [];
  }
}
const QUERY_STOPWORDS = /* @__PURE__ */ new Set([
  "the",
  "a",
  "an",
  "is",
  "are",
  "be",
  "in",
  "of",
  "to",
  "and",
  "or",
  "for",
  "on",
  "at",
  "with",
  "by",
  "as",
  "it",
  "this",
  "that",
  "these",
  "those",
  "how",
  "what",
  "why",
  "when",
  "where",
  "do",
  "does",
  "will",
  "can"
]);
function generateSearchQueries(terms, title) {
  const meaningful = terms.map((t) => t.trim()).filter((t) => t.length >= 3 && !QUERY_STOPWORDS.has(t.toLowerCase()));
  const haveTitleSignal = !!title && title.length > 3;
  if (meaningful.length < 2 && !haveTitleSignal) return [];
  const queries = [];
  if (haveTitleSignal) {
    queries.push(title.slice(0, 150));
  }
  if (meaningful.length >= 2) {
    queries.push(meaningful.slice(0, 5).join(" "));
  }
  if (meaningful.length >= 3) {
    queries.push(`${meaningful.slice(0, 3).join(" ")} overview`);
  }
  const unique = [...new Set(queries)];
  return unique.slice(0, WEB_SEARCH_QUERY_COUNT);
}
function fuseResults(rankedLists, allResults, maxResults) {
  const scores = reciprocalRankFusion(rankedLists);
  const sorted = sortByRRFScore(scores);
  const resultsByNormalizedUrl = /* @__PURE__ */ new Map();
  for (const r of allResults) {
    const key = safeNormalize(r.url);
    if (!resultsByNormalizedUrl.has(key)) {
      resultsByNormalizedUrl.set(key, r);
    }
  }
  const topScore = sorted.length > 0 ? sorted[0][1] : 0;
  const fused = [];
  for (const [nUrl, score] of sorted) {
    if (fused.length >= maxResults) break;
    const result = resultsByNormalizedUrl.get(nUrl);
    if (!result) continue;
    const normalized = topScore > 0 ? score / topScore : 0;
    fused.push({
      ...result,
      relevance_score: normalized,
      match_signals: {
        ...result.match_signals,
        fused_score: score
      }
    });
  }
  return { results: fused, topRawScore: topScore };
}
function determineMethod(hasCache, hasEmbedding, hasSearch) {
  const sources = [hasCache, hasEmbedding, hasSearch].filter(Boolean).length;
  if (sources >= 2) return "hybrid";
  if (hasEmbedding) return "embedding";
  if (hasCache) return "fts5";
  if (hasSearch) return "search";
  return "fts5";
}
export {
  findSimilar,
  generateSearchQueries
};
//# sourceMappingURL=find-similar.js.map