"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.loadCorpusFromUrlsFile = loadCorpusFromUrlsFile;
const promises_1 = require("fs/promises");
const text_1 = require("./text");
const webLoader_1 = require("./webLoader");
const staticPath_1 = require("../utils/staticPath");
async function loadCorpusFromUrlsFile(filePath) {
    const urlsPath = filePath !== null && filePath !== void 0 ? filePath : (0, staticPath_1.resolveStaticFilePath)("docsURLs.txt");
    const urls = (await (0, promises_1.readFile)(urlsPath, "utf-8"))
        .split("\n")
        .map(s => s.trim())
        .filter(Boolean);
    const chunks = [];
    for (const url of urls) {
        const text = await (0, webLoader_1.loadWebText)(url);
        const parts = (0, text_1.chunkText)(text, 500, 100);
        for (let i = 0; i < parts.length; i++) {
            chunks.push({
                id: `${url}#${i}`,
                source: url,
                chunkIndex: i,
                text: parts[i],
            });
        }
    }
    return chunks;
}
