"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
    if (mod && mod.__esModule) return mod;
    var result = {};
    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
    __setModuleDefault(result, mod);
    return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.SourceType = void 0;
const polyfact_1 = require("polyfact");
const lodash_1 = __importDefault(require("lodash"));
const pdf_1 = require("langchain/document_loaders/fs/pdf");
const path = __importStar(require("path"));
const fs_1 = __importDefault(require("fs"));
var SourceType;
(function (SourceType) {
    SourceType[SourceType["DIRECTORY"] = 0] = "DIRECTORY";
    SourceType[SourceType["PDF"] = 1] = "PDF";
})(SourceType = exports.SourceType || (exports.SourceType = {}));
function readDirRecursive(dir) {
    const result = [];
    const list = fs_1.default.readdirSync(dir);
    list.forEach((file) => {
        const filePath = path.resolve(dir, file);
        const stat = fs_1.default.statSync(filePath);
        if (stat && stat.isDirectory()) {
            result.push(...readDirRecursive(filePath));
        }
        else {
            const content = fs_1.default.readFileSync(filePath, { encoding: "utf-8" });
            result.push({ path: filePath, content });
        }
    });
    return result;
}
function batchify(array, size) {
    const batched = [];
    let copied = [...array];
    const numOfChild = Math.ceil(copied.length / size);
    for (let i = 0; i < numOfChild; i++) {
        batched.push(copied.splice(0, size));
    }
    return batched;
}
function splitFiles(files, maxToken, progress) {
    if (files.length === 0) {
        throw new Error("You need to add files to split !");
    }
    const result = [];
    files.forEach((file) => {
        const splittedString = (0, polyfact_1.splitString)(file.content, maxToken);
        if (splittedString.length > 1) {
            splittedString.forEach((s, i) => {
                result.push({ path: `${file.path}_${i}`, content: s });
            });
        }
        else {
            result.push(file);
        }
        progress(file);
    });
    return result;
}
class Vectorizer {
    constructor(api_key, maxToken, sourceType, memoryId) {
        if (maxToken <= 0) {
            throw new Error("maxToken must be a positive number");
        }
        this.apiKey = api_key;
        this.memoryId = memoryId || null;
        this.maxToken = maxToken;
        this.sourceType = sourceType;
    }
    async ensureMemoryInitialized() {
        if (!this.memoryId) {
            this.memoryId = (await (0, polyfact_1.createMemory)()).id;
        }
    }
    async readPDFs(files) {
        const output = [];
        async function extractPDFContent(filePath, perPage = true) {
            const loaderOptions = perPage ? {} : { splitPages: false };
            const loader = new pdf_1.PDFLoader(filePath, loaderOptions);
            try {
                const docs = await loader.load();
                const filename = path.basename(filePath);
                if (perPage) {
                    for (let i = 0; i < docs.length; i++) {
                        output.push({
                            content: docs[i].pageContent,
                            page: i + 1,
                            filename: filename,
                            path: filePath,
                        });
                    }
                }
                else {
                    output.push({
                        content: docs[0].pageContent,
                        page: 1,
                        filename: filename,
                        path: filePath,
                    });
                }
            }
            catch (error) {
                throw new Error(`Error reading "${filePath}": ${error.message}`);
            }
        }
        const promises = files.map((file) => extractPDFContent(path.resolve(file)));
        await Promise.all(promises);
        return output;
    }
    readFilesFromRepo(files) {
        if (!Array.isArray(files)) {
            throw new Error(`Expected an array, but received ${typeof files}`);
        }
        return lodash_1.default.flatten(files.map((item) => {
            const filePath = path.resolve(item);
            if (fs_1.default.existsSync(filePath)) {
                if (fs_1.default.statSync(filePath).isDirectory()) {
                    return readDirRecursive(filePath);
                }
                else {
                    const content = fs_1.default.readFileSync(filePath, { encoding: "utf-8" });
                    return [{ path: filePath, content }];
                }
            }
            else {
                throw new Error(`The path "${filePath}" does not exist.`);
            }
        }));
    }
    async readFiles(files) {
        if (this.sourceType === SourceType.DIRECTORY) {
            return this.readFilesFromRepo(files);
        }
        else if (this.sourceType === SourceType.PDF) {
            return this.readPDFs(files);
        }
        else {
            throw new Error("Unknown source type");
        }
    }
    async vectorize(files, progress) {
        if (files.length === 0) {
            throw new Error("No files provided for vectorization!");
        }
        await this.ensureMemoryInitialized();
        function splitProgess(file) {
            progress(file, files.length, "split");
        }
        function vectorizeProgress(file) {
            progress(file, files.length, "vectorize");
        }
        files = splitFiles(files, this.maxToken, splitProgess);
        const batchs = batchify(files, 10);
        const result = [];
        for (let i = 0; i < batchs.length; i++) {
            const batch = batchs[i];
            await Promise.all(batch.map(async (file) => {
                if (this.memoryId === null) {
                    throw new Error("Memory ID is not set.");
                }
                const updated = await (0, polyfact_1.updateMemory)(this.memoryId, JSON.stringify(file), this.maxToken, { token: this.apiKey });
                file = Object.assign(Object.assign({}, file), { saved: Boolean(updated) });
                result.push(file);
                vectorizeProgress(file);
            }));
        }
        return result;
    }
    getMemoryId() {
        if (this.memoryId === null) {
            throw new Error("Memory ID is not set.");
        }
        return this.memoryId;
    }
}
exports.default = Vectorizer;
//# sourceMappingURL=index.js.map