import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { getMarkdownMetadata } from "@intlayer/core";
import { getBlogs, getDocs, getFrequentQuestions } from "@intlayer/docs";
import { OpenAI } from "openai";
import dotenv from "dotenv";

//#region src/utils/AI/askDocQuestion/indexMarkdownFiles.ts
const OUTPUT_EMBEDDINGS_DIR = "src/utils/AI/askDocQuestion/embeddings";
const __dirname = dirname(fileURLToPath(import.meta.url));
const readEmbeddingsForFile = (fileKey) => {
	try {
		return JSON.parse(readFileSync(`${__dirname}/embeddings/${fileKey.replace(".md", ".json")}`, "utf-8"));
	} catch {
		return {};
	}
};
const writeEmbeddingsForFile = (fileKey, data) => {
	const filePath = join(OUTPUT_EMBEDDINGS_DIR, `${fileKey.replace(".md", ".json")}`);
	const dir = dirname(filePath);
	if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
	writeFileSync(filePath, JSON.stringify(data));
};
/**
* Simple in-memory vector store to hold document embeddings and their content.
* Each entry contains:
* - fileKey: A unique key identifying the file
* - chunkNumber: The number of the chunk within the document
* - content: The chunk content
* - embedding: The numerical embedding vector for the chunk
*/
const vectorStore = [];
const EMBEDDING_MODEL = "text-embedding-3-large";
const OVERLAP_TOKENS = 200;
const MAX_CHUNK_TOKENS = 800;
const CHAR_BY_TOKEN = 4.15;
const MAX_CHARS = MAX_CHUNK_TOKENS * CHAR_BY_TOKEN;
const OVERLAP_CHARS = OVERLAP_TOKENS * CHAR_BY_TOKEN;
const skipDocEmbeddingsIndex = process.env.SKIP_DOC_EMBEDDINGS_INDEX === "true";
/**
* Splits a given text into chunks ensuring each chunk does not exceed MAX_CHARS.
* @param text - The input text to split.
* @returns - Array of text chunks.
*/
const chunkText = (text) => {
	const chunks = [];
	let start = 0;
	while (start < text.length) {
		let end = Math.min(start + MAX_CHARS, text.length);
		if (end < text.length) {
			const lastSpace = text.lastIndexOf(" ", end);
			if (lastSpace > start) end = lastSpace;
		}
		chunks.push(text.substring(start, end));
		const nextStart = end - OVERLAP_CHARS;
		if (nextStart <= start) start = end;
		else start = nextStart;
	}
	return chunks;
};
/**
* Generates an embedding for a given text using OpenAI's embedding API.
* Trims the text if it exceeds the maximum allowed characters.
*
* @param text - The input text to generate an embedding for
* @returns The embedding vector as a number array
*/
const generateEmbedding = async (text) => {
	try {
		return (await new OpenAI({ apiKey: process.env.OPENAI_API_KEY }).embeddings.create({
			model: EMBEDDING_MODEL,
			input: text
		})).data[0].embedding;
	} catch (error) {
		console.error("Error generating embedding:", error);
		return [];
	}
};
/**
* Indexes all Markdown documents by generating embeddings for each chunk and storing them in memory.
* Persists per-document embeddings under `embeddings/<fileKey>.json`.
* Handles cases where files have been updated and chunk counts have changed.
*/
const indexMarkdownFiles = async () => {
	const env = "development";
	dotenv.config({ path: [
		`.env.${env}.local`,
		`.env.${env}`,
		".env.local",
		".env"
	] });
	const frequentQuestions = await getFrequentQuestions();
	const docs = await getDocs();
	const blogs = await getBlogs();
	const files = {
		...docs,
		...blogs,
		...frequentQuestions
	};
	for await (const fileKey of Object.keys(files)) {
		const fileMetadata = getMarkdownMetadata(files[fileKey]);
		const fileChunks = chunkText(files[fileKey]);
		const existingEmbeddings = readEmbeddingsForFile(fileKey);
		const existingChunksForFile = Object.keys(existingEmbeddings);
		const currentChunkCount = fileChunks.length;
		const previousChunkCount = existingChunksForFile.length;
		let shouldRegenerateFileEmbeddings = false;
		if (currentChunkCount !== previousChunkCount) {
			console.info(`File "${fileKey}" chunk count changed: ${previousChunkCount} -> ${currentChunkCount}. Regenerating embeddings.`);
			shouldRegenerateFileEmbeddings = !skipDocEmbeddingsIndex;
		}
		let resultForFile = {};
		for await (const chunkIndex of Object.keys(fileChunks)) {
			const chunkNumber = Number(chunkIndex) + 1;
			const chunksNumber = fileChunks.length;
			const fileChunk = fileChunks[chunkIndex];
			const chunkKeyName = `chunk_${chunkNumber}`;
			let embedding = !shouldRegenerateFileEmbeddings ? existingEmbeddings[chunkKeyName] : void 0;
			if (!embedding) {
				embedding = await generateEmbedding(fileChunk);
				console.info(`- Generated new embedding: ${fileKey}/${chunkKeyName}`);
			}
			resultForFile = {
				...resultForFile,
				[chunkKeyName]: embedding
			};
			vectorStore.push({
				fileKey,
				chunkNumber,
				embedding,
				content: fileChunk,
				docUrl: fileMetadata.url,
				docName: fileMetadata.title
			});
			console.info(`- Indexed: ${fileKey}/${chunkKeyName}/${chunksNumber}`);
		}
		try {
			if (JSON.stringify(resultForFile) !== JSON.stringify(existingEmbeddings)) writeEmbeddingsForFile(fileKey, resultForFile);
		} catch (error) {
			console.error(error);
		}
	}
};

//#endregion
export { indexMarkdownFiles };
//# sourceMappingURL=indexMarkdownFiles.mjs.map