const require_rolldown_runtime = require('../../../_virtual/rolldown_runtime.cjs');
const require_utils_AI_aiSdk = require('../aiSdk.cjs');
let node_fs = require("node:fs");
let node_path = require("node:path");
let node_url = require("node:url");
let __intlayer_core = require("@intlayer/core");
let __intlayer_docs = require("@intlayer/docs");
let openai = require("openai");
let dotenv = require("dotenv");
dotenv = require_rolldown_runtime.__toESM(dotenv);

//#region src/utils/AI/askDocQuestion/indexMarkdownFiles.ts
const OUTPUT_EMBEDDINGS_DIR = "src/utils/AI/askDocQuestion/embeddings";
const __dirname$1 = (0, node_path.dirname)((0, node_url.fileURLToPath)(require("url").pathToFileURL(__filename).href));
const readEmbeddingsForFile = (fileKey) => {
	try {
		return JSON.parse((0, node_fs.readFileSync)(`${__dirname$1}/embeddings/${fileKey.replace(".md", ".json")}`, "utf-8"));
	} catch {
		return {};
	}
};
const writeEmbeddingsForFile = (fileKey, data) => {
	const filePath = (0, node_path.join)(OUTPUT_EMBEDDINGS_DIR, `${fileKey.replace(".md", ".json")}`);
	const dir = (0, node_path.dirname)(filePath);
	if (!(0, node_fs.existsSync)(dir)) (0, node_fs.mkdirSync)(dir, { recursive: true });
	(0, node_fs.writeFileSync)(filePath, JSON.stringify(data));
};
/**
* Simple in-memory vector store to hold document embeddings and their content.
* Each entry contains:
* - fileKey: A unique key identifying the file
* - chunkNumber: The number of the chunk within the document
* - content: The chunk content
* - embedding: The numerical embedding vector for the chunk
*/
const vectorStore = [];
const MODEL = "chatgpt-4o-latest";
const MODEL_TEMPERATURE = .1;
const aiDefaultOptions = {
	provider: require_utils_AI_aiSdk.AIProvider.OPENAI,
	model: MODEL,
	temperature: MODEL_TEMPERATURE
};
const EMBEDDING_MODEL = "text-embedding-3-large";
const OVERLAP_TOKENS = 200;
const MAX_CHUNK_TOKENS = 800;
const CHAR_BY_TOKEN = 4.15;
const MAX_CHARS = MAX_CHUNK_TOKENS * CHAR_BY_TOKEN;
const OVERLAP_CHARS = OVERLAP_TOKENS * CHAR_BY_TOKEN;
const skipDocEmbeddingsIndex = process.env.SKIP_DOC_EMBEDDINGS_INDEX === "true";
/**
* Splits a given text into chunks ensuring each chunk does not exceed MAX_CHARS.
* @param text - The input text to split.
* @returns - Array of text chunks.
*/
const chunkText = (text) => {
	const chunks = [];
	let start = 0;
	while (start < text.length) {
		let end = Math.min(start + MAX_CHARS, text.length);
		if (end < text.length) {
			const lastSpace = text.lastIndexOf(" ", end);
			if (lastSpace > start) end = lastSpace;
		}
		chunks.push(text.substring(start, end));
		const nextStart = end - OVERLAP_CHARS;
		if (nextStart <= start) start = end;
		else start = nextStart;
	}
	return chunks;
};
/**
* Generates an embedding for a given text using OpenAI's embedding API.
* Trims the text if it exceeds the maximum allowed characters.
*
* @param text - The input text to generate an embedding for
* @returns The embedding vector as a number array
*/
const generateEmbedding = async (text) => {
	try {
		return (await new openai.OpenAI({ apiKey: process.env.OPENAI_API_KEY }).embeddings.create({
			model: EMBEDDING_MODEL,
			input: text
		})).data[0].embedding;
	} catch (error) {
		console.error("Error generating embedding:", error);
		return [];
	}
};
/**
* Indexes all Markdown documents by generating embeddings for each chunk and storing them in memory.
* Persists per-document embeddings under `embeddings/<fileKey>.json`.
* Handles cases where files have been updated and chunk counts have changed.
*/
const indexMarkdownFiles = async () => {
	const env = process.env.NODE_ENV;
	dotenv.default.config({ path: [
		`.env.${env}.local`,
		`.env.${env}`,
		".env.local",
		".env"
	] });
	const frequentQuestions = await (0, __intlayer_docs.getFrequentQuestions)();
	const docs = await (0, __intlayer_docs.getDocs)();
	const blogs = await (0, __intlayer_docs.getBlogs)();
	const files = {
		...docs,
		...blogs,
		...frequentQuestions
	};
	for await (const fileKey of Object.keys(files)) {
		const fileMetadata = (0, __intlayer_core.getMarkdownMetadata)(files[fileKey]);
		const fileChunks = chunkText(files[fileKey]);
		const existingEmbeddings = readEmbeddingsForFile(fileKey);
		const existingChunksForFile = Object.keys(existingEmbeddings);
		const currentChunkCount = fileChunks.length;
		const previousChunkCount = existingChunksForFile.length;
		let shouldRegenerateFileEmbeddings = false;
		if (currentChunkCount !== previousChunkCount) {
			console.info(`File "${fileKey}" chunk count changed: ${previousChunkCount} -> ${currentChunkCount}. Regenerating embeddings.`);
			shouldRegenerateFileEmbeddings = !skipDocEmbeddingsIndex;
		}
		let resultForFile = {};
		for await (const chunkIndex of Object.keys(fileChunks)) {
			const chunkNumber = Number(chunkIndex) + 1;
			const chunksNumber = fileChunks.length;
			const fileChunk = fileChunks[chunkIndex];
			const chunkKeyName = `chunk_${chunkNumber}`;
			let embedding = !shouldRegenerateFileEmbeddings ? existingEmbeddings[chunkKeyName] : void 0;
			if (!embedding) {
				embedding = await generateEmbedding(fileChunk);
				console.info(`- Generated new embedding: ${fileKey}/${chunkKeyName}`);
			}
			resultForFile = {
				...resultForFile,
				[chunkKeyName]: embedding
			};
			vectorStore.push({
				fileKey,
				chunkNumber,
				embedding,
				content: fileChunk,
				docUrl: fileMetadata.url,
				docName: fileMetadata.title
			});
			console.info(`- Indexed: ${fileKey}/${chunkKeyName}/${chunksNumber}`);
		}
		try {
			if (JSON.stringify(resultForFile) !== JSON.stringify(existingEmbeddings)) writeEmbeddingsForFile(fileKey, resultForFile);
		} catch (error) {
			console.error(error);
		}
	}
};

//#endregion
exports.aiDefaultOptions = aiDefaultOptions;
exports.indexMarkdownFiles = indexMarkdownFiles;
//# sourceMappingURL=indexMarkdownFiles.cjs.map