All files / Cognigy-CLI/build/lib/knowledgeAI handleIngest.js

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.handleIngest = void 0;
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const config_1 = __importDefault(require("../../utils/config"));
const ingestFile_1 = require("./ingestFile");
const cli_progress_1 = __importDefault(require("cli-progress"));
const handleIngest = async (knowledgeStoreId, input, name) => {
    console.clear();
    const bar = new cli_progress_1.default.SingleBar({
        format: ' {bar} | {value}/{total}',
        hideCursor: true,
    }, cli_progress_1.default.Presets.shades_classic);
    let errors = [];
    if (name) {
        const config = fs_1.default.readFileSync(`${config_1.default.knowledgeAIStoreDir}/knowledgeStore_${name}.json`, 'utf-8');
        if (config) {
            try {
                const parsedConfig = JSON.parse(config);
                knowledgeStoreId = parsedConfig._id;
                console.log(`Successfully read config for knowledge store '${name}' and using it for ingestion...`);
            }
            catch (err) { }
        }
        else {
            console.log(`Config file knowledgeStore_${name}.json not found. Aborting...`);
            process.exit(1);
        }
    }
    const stats = fs_1.default.statSync(input);
    if (stats.isFile() && path_1.default.extname(input) === '.ctxt') {
        console.log(`Processing file: ${input}`);
        bar.start(1, 0);
        await (0, ingestFile_1.ingestFile)(knowledgeStoreId, input);
        bar.increment();
        bar.stop();
    }
    else if (stats.isDirectory()) {
        let files = fs_1.default.readdirSync(input);
        files = files.filter((f) => f.endsWith('.ctxt'));
        const numFiles = files.length;
        let index = 0;
        bar.start(numFiles, 0);
        for (const file of files) {
            index += 1;
            const filePath = path_1.default.join(input, file);
            const fileStats = fs_1.default.statSync(filePath);
            let fileExtension = path_1.default.extname(filePath);
            console.log(`\nProcessing file ${index} of ${numFiles}: ${filePath}`);
            if (fileStats.isFile() && fileExtension === '.ctxt') {
                await (0, ingestFile_1.ingestFile)(knowledgeStoreId, filePath);
                console.clear();
            }
            else {
                if (fileExtension === '') {
                    fileExtension = 'directory';
                }
                console.log(`File format "${fileExtension}" is not supported`);
            }
            bar.increment();
        }
        bar.stop();
        console.log(`All ${numFiles} files,\n${JSON.stringify(files, null, 2)}\nfrom directory ${input} have been successfully processed!`);
    }
    else {
        throw Error('Input must be a text file or a directory containing text files');
    }
    if (errors.length > 0) {
        console.warn('Some errors were thrown during execution. See the following list.');
        for (let error of errors) {
            console.warn('-', error);
        }
    }
};
exports.handleIngest = handleIngest;