import { createHash } from "crypto";
import { createLogger } from "../logger.js";
import { getVectorStore } from "../providers/vector-store.js";
import { getEmbedProvider } from "../providers/embed-provider.js";
import { getBackgroundIndexQueue } from "../embedding/background-queue.js";
const log = createLogger("crawl");
const SUMMARY_CHARS = 500;
const MIN_TEXT_LEN = 20;
function isIndexingEnabled() {
  return process.env.WIGOLO_CRAWL_INDEX === "1";
}
function summariseForIndex(item) {
  const summary = (item.markdown ?? "").slice(0, SUMMARY_CHARS);
  const text = `${item.title ?? ""}
${summary}`.trim();
  if (text.length < MIN_TEXT_LEN) return null;
  const contentHash = createHash("sha256").update(item.markdown ?? "").digest("hex");
  return { text, contentHash };
}
function enqueueIndexCrawl(item) {
  const summary = summariseForIndex(item);
  if (!summary) return Promise.resolve();
  return getBackgroundIndexQueue().enqueue({
    url: item.url,
    text: summary.text,
    contentHash: summary.contentHash
  });
}
async function indexCrawlResult(item) {
  try {
    const summary = (item.markdown ?? "").slice(0, SUMMARY_CHARS);
    const text = `${item.title ?? ""}
${summary}`.trim();
    if (text.length < MIN_TEXT_LEN) return;
    const provider = await getEmbedProvider();
    const vectors = await provider.embed([text]);
    if (vectors.length === 0) return;
    const store = await getVectorStore();
    const contentHash = createHash("sha256").update(item.markdown ?? "").digest("hex");
    await store.upsert([
      {
        id: item.url,
        vector: vectors[0],
        metadata: {
          url: item.url,
          contentHash,
          modelId: provider.modelId
        }
      }
    ]);
  } catch (err) {
    log.warn("crawl index-to-vec failed", {
      url: item.url,
      error: err instanceof Error ? err.message : String(err)
    });
  }
}
export {
  enqueueIndexCrawl,
  indexCrawlResult,
  isIndexingEnabled
};
//# sourceMappingURL=index-to-vec.js.map