import{logger as e}from"../../../logger/index.mjs";import{existsSync as t,mkdirSync as n,readFileSync as r,writeFileSync as i}from"node:fs";import{dirname as a,join as o}from"node:path";import{fileURLToPath as s}from"node:url";import{getMarkdownMetadata as c}from"@intlayer/core/markdown";import{getBlogs as l,getDocs as u,getFrequentQuestions as d}from"@intlayer/docs";import{OpenAI as f}from"openai";import p from"dotenv";const m=a(s(import.meta.url)),h=e=>{try{return JSON.parse(r(`${m}/embeddings/${e.replace(`.md`,`.json`)}`,`utf-8`))}catch{return{}}},g=(e,r)=>{let s=o(`src/utils/AI/askDocQuestion/embeddings`,`${e.replace(`.md`,`.json`)}`),c=a(s);t(c)||n(c,{recursive:!0}),i(s,JSON.stringify(r))},_=[],v=4.15;800*v,200*v;const y=process.env.SKIP_DOC_EMBEDDINGS_INDEX===`true`,b=e=>{let t=[],n=0;for(;n<e.length;){let r=Math.min(n+3320.0000000000005,e.length);if(r<e.length){let t=e.lastIndexOf(` `,r);t>n&&(r=t)}t.push(e.substring(n,r));let i=r-830.0000000000001;n=i<=n?r:i}return t},x=async t=>{try{return(await new f({apiKey:process.env.OPENAI_API_KEY}).embeddings.create({model:`text-embedding-3-large`,input:t})).data[0].embedding}catch(t){return e.error(`Error generating embedding:`,t),[]}},S=async()=>{let t=`production`;p.config({path:[`.env.${t}.local`,`.env.${t}`,`.env.local`,`.env`],quiet:!0});let n=await d(),r=await u(),i=await l(),a={...r,...i,...n};for await(let t of Object.keys(a)){let n=c(a[t]),r=b(a[t]),i=h(t),o=Object.keys(i),s=r.length,l=o.length,u=!1;s!==l&&(e.info(`File "${t}" chunk count changed: ${l} -> ${s}. Regenerating embeddings.`),u=!y);let d={};for await(let a of Object.keys(r)){let o=Number(a)+1,s=r.length,c=r[a],l=`chunk_${o}`,f=u?void 0:i[l];f||(f=await x(c),e.info(`- Generated new embedding: ${t}/${l}`)),d={...d,[l]:f},_.push({fileKey:t,chunkNumber:o,embedding:f,content:c,docUrl:n.url,docName:n.title}),e.info(`- Indexed: ${t}/${l}/${s}`)}try{JSON.stringify(d)!==JSON.stringify(i)&&g(t,d)}catch(t){e.error(t)}}};export{S as indexMarkdownFiles};
//# sourceMappingURL=indexMarkdownFiles.mjs.map