import{existsSync as e,mkdirSync as t,readFileSync as n,writeFileSync as r}from"node:fs";import{dirname as i,join as a}from"node:path";import{fileURLToPath as o}from"node:url";import{getMarkdownMetadata as s}from"@intlayer/core/markdown";import{getBlogs as c,getDocs as l,getFrequentQuestions as u}from"@intlayer/docs";import{OpenAI as d}from"openai";import f from"dotenv";const p=i(o(import.meta.url)),m=e=>{try{return JSON.parse(n(`${p}/embeddings/${e.replace(`.md`,`.json`)}`,`utf-8`))}catch{return{}}},h=(n,o)=>{let s=a(`src/utils/AI/askDocQuestion/embeddings`,`${n.replace(`.md`,`.json`)}`),c=i(s);e(c)||t(c,{recursive:!0}),r(s,JSON.stringify(o))},g=[],_=4.15;800*_,200*_;const v=process.env.SKIP_DOC_EMBEDDINGS_INDEX===`true`,y=e=>{let t=[],n=0;for(;n<e.length;){let r=Math.min(n+3320.0000000000005,e.length);if(r<e.length){let t=e.lastIndexOf(` `,r);t>n&&(r=t)}t.push(e.substring(n,r));let i=r-830.0000000000001;n=i<=n?r:i}return t},b=async e=>{try{return(await new d({apiKey:process.env.OPENAI_API_KEY}).embeddings.create({model:`text-embedding-3-large`,input:e})).data[0].embedding}catch(e){return console.error(`Error generating embedding:`,e),[]}},x=async()=>{let e=`production`;f.config({path:[`.env.${e}.local`,`.env.${e}`,`.env.local`,`.env`],quiet:!0});let t=await u(),n=await l(),r=await c(),i={...n,...r,...t};for await(let e of Object.keys(i)){let t=s(i[e]),n=y(i[e]),r=m(e),a=Object.keys(r),o=n.length,c=a.length,l=!1;o!==c&&(console.info(`File "${e}" chunk count changed: ${c} -> ${o}. Regenerating embeddings.`),l=!v);let u={};for await(let i of Object.keys(n)){let a=Number(i)+1,o=n.length,s=n[i],c=`chunk_${a}`,d=l?void 0:r[c];d||(d=await b(s),console.info(`- Generated new embedding: ${e}/${c}`)),u={...u,[c]:d},g.push({fileKey:e,chunkNumber:a,embedding:d,content:s,docUrl:t.url,docName:t.title}),console.info(`- Indexed: ${e}/${c}/${o}`)}try{JSON.stringify(u)!==JSON.stringify(r)&&h(e,u)}catch(e){console.error(e)}}};export{x as indexMarkdownFiles};
//# sourceMappingURL=indexMarkdownFiles.mjs.map