"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
    var ownKeys = function(o) {
        ownKeys = Object.getOwnPropertyNames || function (o) {
            var ar = [];
            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
            return ar;
        };
        return ownKeys(o);
    };
    return function (mod) {
        if (mod && mod.__esModule) return mod;
        var result = {};
        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
        __setModuleDefault(result, mod);
        return result;
    };
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.resolveDatasetUploadDataType = resolveDatasetUploadDataType;
exports.resolveBulkAppendMode = resolveBulkAppendMode;
exports.resolveZipKind = resolveZipKind;
exports.getVariableColumnName = getVariableColumnName;
exports.extractZipToDir = extractZipToDir;
exports.cleanupDir = cleanupDir;
exports.cleanupFile = cleanupFile;
exports.previewZipContents = previewZipContents;
exports.previewCsvForDataset = previewCsvForDataset;
exports.assertAppendWithinLimit = assertAppendWithinLimit;
exports.appendDatasetRecords = appendDatasetRecords;
exports.commitZipAppend = commitZipAppend;
exports.commitCsvAppend = commitCsvAppend;
exports.loadDatasetForBulkAppend = loadDatasetForBulkAppend;
const drizzle_orm_1 = require("drizzle-orm");
const extract_zip_1 = __importDefault(require("extract-zip"));
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const sync_1 = require("csv-parse/sync");
const uuid_1 = require("uuid");
const constants_1 = require("../../../constants");
const database_1 = require("../../../database");
const tables_1 = require("../../../database/tables");
const logger_1 = __importDefault(require("../../../logger"));
const uploadFilesToAzure_1 = require("../../../util/uploadFilesToAzure");
const datasetRowsService_1 = require("./datasetRowsService");
const datasetVoiceTranscript_1 = require("./datasetVoiceTranscript");
/** Max parallel I/O-bound steps (ingest, transcribe, normalize) for bulk import — balances latency vs upstream rate limits. */
const BULK_IO_CONCURRENCY = 8;
/**
 * Map `items` with at most `concurrency` concurrent `fn` calls; results match original order.
 */
async function mapWithConcurrencyLimit(items, concurrency, fn) {
    if (items.length === 0) {
        return [];
    }
    const results = new Array(items.length);
    const limit = Math.min(Math.max(1, concurrency), items.length);
    let next = 0;
    const worker = async () => {
        for (;;) {
            const i = next++;
            if (i >= items.length) {
                return;
            }
            results[i] = await fn(items[i], i);
        }
    };
    await Promise.all(Array.from({ length: limit }, () => worker()));
    return results;
}
const getFieldName = (f) => typeof f === 'object' ? String(f.name ?? '') : String(f);
const getFieldType = (f) => typeof f === 'object' && f.type ? String(f.type) : constants_1.DATASET_VAR_TYPE.TEXT;
/** Same source of truth as dataset create/edit UI: uploadDataType first, then dataType. */
function resolveDatasetUploadDataType(config) {
    const c = config;
    const dataType = Number(c?.uploadDataType ?? c?.dataType);
    return Number.isFinite(dataType) ? dataType : null;
}
function resolveBulkAppendMode(dataset) {
    if (dataset.type === constants_1.DATASET_TYPE.UPLOAD) {
        const dataType = resolveDatasetUploadDataType(dataset.config);
        if (dataType == null) {
            return null;
        }
        if (dataType === constants_1.DATASET_UPLOAD_DATA_TYPE.AUDIO ||
            dataType === constants_1.DATASET_UPLOAD_DATA_TYPE.VIDEO ||
            dataType === constants_1.DATASET_UPLOAD_DATA_TYPE.IMAGE ||
            dataType === constants_1.DATASET_UPLOAD_DATA_TYPE.CONVERSATION) {
            return 'zip';
        }
        return 'csv';
    }
    if (dataset.type === constants_1.DATASET_TYPE.GENERATE || dataset.type === constants_1.DATASET_TYPE.GENERATE_FROM_DOCUMENTS) {
        return 'csv';
    }
    return null;
}
function resolveZipKind(dataType) {
    if (dataType === constants_1.DATASET_UPLOAD_DATA_TYPE.AUDIO) {
        return 'audio';
    }
    if (dataType === constants_1.DATASET_UPLOAD_DATA_TYPE.VIDEO) {
        return 'video';
    }
    if (dataType === constants_1.DATASET_UPLOAD_DATA_TYPE.IMAGE) {
        return 'image';
    }
    if (dataType === constants_1.DATASET_UPLOAD_DATA_TYPE.CONVERSATION) {
        return 'conversation';
    }
    return null;
}
function listFilesRecursive(dir) {
    const out = [];
    if (!fs.existsSync(dir)) {
        return out;
    }
    const walk = (d) => {
        const entries = fs.readdirSync(d);
        for (const name of entries) {
            const full = path.join(d, name);
            const stat = fs.statSync(full);
            if (stat.isDirectory()) {
                walk(full);
            }
            else {
                out.push(full);
            }
        }
    };
    walk(dir);
    return out;
}
const getFileExtension = (filename) => path.extname(filename).toLowerCase();
const getContentType = (extension) => {
    const contentTypes = {
        '.mp4': 'video/mp4',
        '.mpg': 'video/mpeg',
        '.mpeg': 'video/mpeg',
        '.webm': 'video/webm',
        '.avi': 'video/x-msvideo',
        '.jpg': 'image/jpeg',
        '.jpeg': 'image/jpeg',
        '.png': 'image/png',
        '.webp': 'image/webp',
        '.txt': 'text/plain',
        '.json': 'application/json',
        '.log': 'text/plain',
    };
    return contentTypes[extension] || 'application/octet-stream';
};
const isValidOpenAIConversation = (content) => {
    try {
        const trimmedContent = content.trim();
        if (!trimmedContent) {
            return false;
        }
        const parsed = JSON.parse(trimmedContent);
        if (!Array.isArray(parsed) || parsed.length === 0) {
            return false;
        }
        for (const message of parsed) {
            if (typeof message !== 'object' || message === null) {
                return false;
            }
            const keys = Object.keys(message);
            if (keys.length !== 2 || !keys.includes('role') || !keys.includes('content')) {
                return false;
            }
            const m = message;
            if (typeof m.role !== 'string' || typeof m.content !== 'string') {
                return false;
            }
            const validRoles = ['system', 'user', 'assistant'];
            if (!validRoles.includes(m.role) || m.content.trim() === '') {
                return false;
            }
        }
        return true;
    }
    catch {
        return false;
    }
};
function analyzeZipAudio(extractPath) {
    const maxBytes = (0, constants_1.getDatasetAudioMaxFileBytes)();
    const maxMb = constants_1.DATASET_DATA_TYPE_CONFIG[constants_1.DATASET_UPLOAD_DATA_TYPE.AUDIO].maxSize;
    const all = listFilesRecursive(extractPath);
    const validPaths = [];
    let invalidCount = 0;
    let oversizedCount = 0;
    for (const filePath of all) {
        const ext = path.extname(filePath).toLowerCase();
        if ((0, constants_1.isAllowedDatasetAudioExtension)(ext)) {
            let size = 0;
            try {
                size = fs.statSync(filePath).size;
            }
            catch {
                invalidCount += 1;
                continue;
            }
            if (size > maxBytes) {
                oversizedCount += 1;
                continue;
            }
            validPaths.push(filePath);
        }
        else {
            invalidCount += 1;
        }
    }
    const blockingError = oversizedCount > 0
        ? `${oversizedCount} audio file(s) exceed the ${maxMb}MB per-file limit. Remove or split them and try again.`
        : undefined;
    return { validPaths, invalidCount, blockingError };
}
function analyzeZipMedia(extractPath, zipKind) {
    let allowedExtensions = [];
    if (zipKind === 'video') {
        allowedExtensions = ['.mp4', '.mpg', '.mpeg', '.webm', '.avi'];
    }
    else if (zipKind === 'image') {
        allowedExtensions = ['.jpg', '.jpeg', '.png', '.webp'];
    }
    else {
        allowedExtensions = ['.txt', '.json', '.log'];
    }
    const all = listFilesRecursive(extractPath);
    const validPaths = [];
    let invalidCount = 0;
    for (const filePath of all) {
        const ext = getFileExtension(filePath);
        if (!allowedExtensions.includes(ext)) {
            invalidCount += 1;
            continue;
        }
        if (zipKind === 'conversation') {
            try {
                const content = fs.readFileSync(filePath, 'utf-8');
                if (isValidOpenAIConversation(content)) {
                    validPaths.push(filePath);
                }
                else {
                    invalidCount += 1;
                }
            }
            catch {
                invalidCount += 1;
            }
        }
        else {
            validPaths.push(filePath);
        }
    }
    return { validPaths, invalidCount };
}
function getVariableColumnName(dataset) {
    const cfg = dataset.config;
    if (cfg?.variable_name) {
        return String(cfg.variable_name);
    }
    const fields = Array.isArray(dataset.fields) ? dataset.fields : [];
    const first = fields[0];
    return first ? getFieldName(first) : '';
}
function shouldGenerateSasForUploadAppend(dataset) {
    if (dataset.type !== constants_1.DATASET_TYPE.UPLOAD) {
        return false;
    }
    const dataType = resolveDatasetUploadDataType(dataset.config);
    if (dataType == null) {
        return false;
    }
    if (dataType === constants_1.DATASET_UPLOAD_DATA_TYPE.AUDIO || dataType === constants_1.DATASET_UPLOAD_DATA_TYPE.TEXT) {
        return false;
    }
    return true;
}
async function extractZipToDir(zipPath, baseTempDir) {
    const extractPath = path.resolve(baseTempDir, (0, uuid_1.v4)());
    fs.mkdirSync(extractPath, { recursive: true });
    await (0, extract_zip_1.default)(zipPath, { dir: extractPath });
    return extractPath;
}
function cleanupDir(dir) {
    if (!dir || !fs.existsSync(dir)) {
        return;
    }
    try {
        fs.rmSync(dir, { recursive: true, force: true });
    }
    catch (e) {
        logger_1.default.warn(`cleanupDir failed: ${e}`);
    }
}
function cleanupFile(filePath) {
    if (!filePath || !fs.existsSync(filePath)) {
        return;
    }
    try {
        fs.unlinkSync(filePath);
    }
    catch (e) {
        logger_1.default.warn(`cleanupFile failed: ${e}`);
    }
}
function previewZipContents(extractPath, zipKind) {
    if (zipKind === 'audio') {
        const r = analyzeZipAudio(extractPath);
        return {
            validCount: r.validPaths.length,
            invalidCount: r.invalidCount,
            validPaths: r.validPaths,
            blockingError: r.blockingError,
        };
    }
    const r = analyzeZipMedia(extractPath, zipKind);
    return {
        validCount: r.validPaths.length,
        invalidCount: r.invalidCount,
        validPaths: r.validPaths,
    };
}
function previewCsvForDataset(csvText, fields) {
    const fieldNames = fields.map(getFieldName).filter(Boolean);
    if (fieldNames.length === 0) {
        return { validCount: 0, invalidCount: 0, headerError: 'Dataset has no columns', validRows: [] };
    }
    let records;
    try {
        records = (0, sync_1.parse)(csvText, {
            columns: true,
            skip_empty_lines: true,
            trim: true,
            relax_column_count: true,
        });
    }
    catch (e) {
        return {
            validCount: 0,
            invalidCount: 0,
            headerError: e instanceof Error ? e.message : 'Invalid CSV',
            validRows: [],
        };
    }
    if (records.length === 0) {
        return { validCount: 0, invalidCount: 0, headerError: 'CSV has no data rows', validRows: [] };
    }
    const firstKeys = Object.keys(records[0] ?? {});
    const missing = fieldNames.filter((n) => !firstKeys.includes(n));
    if (missing.length > 0) {
        return {
            validCount: 0,
            invalidCount: 0,
            headerError: `Missing required column(s): ${missing.join(', ')}`,
            validRows: [],
        };
    }
    const validRows = [];
    let invalidCount = 0;
    for (const row of records) {
        let ok = true;
        for (const n of fieldNames) {
            const v = row[n];
            if (v === undefined || v === null || String(v).trim() === '') {
                ok = false;
                break;
            }
        }
        if (ok) {
            validRows.push(row);
        }
        else {
            invalidCount += 1;
        }
    }
    return { validCount: validRows.length, invalidCount, headerError: null, validRows };
}
function csvRowToRecordData(row, fields) {
    const out = {};
    for (const f of fields) {
        const name = getFieldName(f);
        if (!name) {
            continue;
        }
        const typ = getFieldType(f);
        const raw = String(row[name] ?? '');
        if (typ === constants_1.DATASET_VAR_TYPE.TEXT) {
            out[name] = { type: constants_1.DATASET_VAR_TYPE.TEXT, value: raw };
        }
        else if (typ === constants_1.DATASET_VAR_TYPE.VOICE) {
            out[name] = { type: constants_1.DATASET_VAR_TYPE.VOICE, value: '', voiceUrl: raw };
        }
        else if (typ === constants_1.DATASET_VAR_TYPE.IMAGE) {
            out[name] = { type: constants_1.DATASET_VAR_TYPE.IMAGE, value: '', imageUrl: raw };
        }
        else if (typ === constants_1.DATASET_VAR_TYPE.VIDEO) {
            out[name] = { type: constants_1.DATASET_VAR_TYPE.VIDEO, value: '', videoUrl: raw };
        }
        else if (typ === constants_1.DATASET_VAR_TYPE.CONVERSATION) {
            out[name] = { type: constants_1.DATASET_VAR_TYPE.CONVERSATION, value: '', conversationUrl: raw };
        }
        else {
            out[name] = { type: constants_1.DATASET_VAR_TYPE.TEXT, value: raw };
        }
    }
    return out;
}
async function getNextRecordStartIndex(datasetId) {
    const db = await (0, database_1.getDb)();
    const maxRecord = await db
        .select({ recordIndex: tables_1.datasetRecordsTable.recordIndex })
        .from(tables_1.datasetRecordsTable)
        .where((0, drizzle_orm_1.eq)(tables_1.datasetRecordsTable.datasetId, datasetId))
        .orderBy((0, drizzle_orm_1.desc)(tables_1.datasetRecordsTable.recordIndex))
        .limit(1);
    return maxRecord.length > 0 ? maxRecord[0].recordIndex + 1 : 0;
}
async function countDatasetRecords(datasetId) {
    const db = await (0, database_1.getDb)();
    const rows = await db
        .select({ count: (0, drizzle_orm_1.sql) `count(*)` })
        .from(tables_1.datasetRecordsTable)
        .where((0, drizzle_orm_1.eq)(tables_1.datasetRecordsTable.datasetId, datasetId));
    return Number(rows[0]?.count ?? 0);
}
function maxRecordsForUploadDataset(dataset) {
    if (dataset.type !== constants_1.DATASET_TYPE.UPLOAD) {
        return null;
    }
    const dataType = resolveDatasetUploadDataType(dataset.config);
    if (dataType == null) {
        return null;
    }
    const cfg = constants_1.DATASET_DATA_TYPE_CONFIG[dataType];
    return cfg?.maxRecords ?? null;
}
async function assertAppendWithinLimit(params) {
    const maxR = maxRecordsForUploadDataset(params.dataset);
    if (maxR == null) {
        return;
    }
    const current = await countDatasetRecords(params.datasetId);
    if (current + params.rowsToAdd > maxR) {
        const err = new Error(`Cannot add ${params.rowsToAdd} row(s): dataset would exceed maximum of ${maxR} records (currently ${current}).`);
        err.status = 400;
        throw err;
    }
}
async function appendDatasetRecords(datasetId, recordDataList) {
    if (recordDataList.length === 0) {
        return 0;
    }
    const db = await (0, database_1.getDb)();
    const start = await getNextRecordStartIndex(datasetId);
    const inserts = recordDataList.map((recordData, i) => ({
        id: (0, uuid_1.v4)(),
        datasetId,
        recordData,
        recordIndex: start + i,
    }));
    await db.insert(tables_1.datasetRecordsTable).values(inserts);
    return inserts.length;
}
async function commitZipAppend(params) {
    const { dataset, datasetId, token, orgId, zipPath, zipKind } = params;
    const tempBase = path.resolve('temp_bulk_append');
    if (!fs.existsSync(tempBase)) {
        fs.mkdirSync(tempBase, { recursive: true });
    }
    let extractPath;
    try {
        extractPath = await extractZipToDir(zipPath, tempBase);
        const preview = previewZipContents(extractPath, zipKind);
        if (preview.blockingError) {
            const err = new Error(preview.blockingError);
            err.status = 400;
            throw err;
        }
        if (preview.validCount === 0) {
            const err = new Error('No valid files to import');
            err.status = 400;
            throw err;
        }
        await assertAppendWithinLimit({
            dataset,
            datasetId,
            rowsToAdd: preview.validCount,
        });
        const variableName = getVariableColumnName(dataset);
        if (!variableName) {
            const err = new Error('Dataset variable column could not be determined');
            err.status = 400;
            throw err;
        }
        const folder = `workspace/${orgId}/project/${dataset.projectId}/dataset/${datasetId}/media`;
        const generateSas = shouldGenerateSasForUploadAppend(dataset);
        if (zipKind === 'audio') {
            const transcripts = await mapWithConcurrencyLimit(preview.validPaths, BULK_IO_CONCURRENCY, (filePath) => (0, datasetVoiceTranscript_1.transcribeDatasetAudioFile)(filePath, dataset.workspaceId));
            const filesToUpload = preview.validPaths.map((filePath, index) => ({
                wavFilePath: filePath,
                fileName: `${index + 1}-${path.basename(filePath)}`,
            }));
            const batchSize = 10;
            const urlMap = {};
            for (let i = 0; i < filesToUpload.length; i += batchSize) {
                const batch = filesToUpload.slice(i, i + batchSize);
                const part = await (0, uploadFilesToAzure_1.uploadAudioFiles)(batch, token, folder);
                Object.assign(urlMap, part);
            }
            const recordDataList = preview.validPaths.map((filePath, index) => {
                const fn = `${index + 1}-${path.basename(filePath)}`;
                const baseName = path.basename(filePath);
                const fallbackName = path.parse(baseName).name;
                const valueText = transcripts[index]?.trim() || fallbackName;
                return {
                    [variableName]: {
                        type: constants_1.DATASET_VAR_TYPE.VOICE,
                        value: valueText,
                        voiceUrl: urlMap[fn] || null,
                    },
                };
            });
            const added = await appendDatasetRecords(datasetId, recordDataList);
            return { addedCount: added };
        }
        const mediaType = zipKind === 'video' ? 'video' : zipKind === 'image' ? 'image' : 'conversation';
        const urlKey = `${mediaType}Url`;
        const filesToUpload = preview.validPaths.map((filePath, index) => {
            const extension = getFileExtension(filePath);
            const contentType = getContentType(extension);
            return {
                filePath,
                fileName: `${index + 1}-${path.basename(filePath)}`,
                contentType,
                cleanupAfterUpload: true,
                privateContainer: true,
                generateSasToken: generateSas,
            };
        });
        const batchSize = 10;
        const urlMap = {};
        for (let i = 0; i < filesToUpload.length; i += batchSize) {
            const batch = filesToUpload.slice(i, i + batchSize);
            const part = await (0, uploadFilesToAzure_1.uploadFilesToAzure)(batch, token, folder);
            Object.assign(urlMap, part);
        }
        const recordDataList = preview.validPaths.map((filePath, index) => {
            const fileName = path.basename(filePath);
            const extension = getFileExtension(filePath);
            const mapKey = `${index + 1}-${fileName}`;
            const mediaUrl = urlMap[mapKey] || null;
            return {
                [variableName]: {
                    type: mediaType,
                    value: path.basename(filePath, extension),
                    [urlKey]: mediaUrl,
                    extension: extension.replace(/^\./, ''),
                    originalName: fileName,
                },
            };
        });
        const added = await appendDatasetRecords(datasetId, recordDataList);
        return { addedCount: added };
    }
    finally {
        cleanupDir(extractPath);
        cleanupFile(zipPath);
    }
}
async function commitCsvAppend(params) {
    const { datasetId, dataset, validRows, token } = params;
    if (validRows.length === 0) {
        const err = new Error('No valid rows to import');
        err.status = 400;
        throw err;
    }
    await assertAppendWithinLimit({
        dataset,
        datasetId,
        rowsToAdd: validRows.length,
    });
    const fields = Array.isArray(dataset.fields) ? dataset.fields : [];
    const datasetCached = {
        id: dataset.id,
        name: '',
        fields: dataset.fields,
        projectId: dataset.projectId,
        workspaceId: dataset.workspaceId,
        type: dataset.type,
        config: dataset.config,
    };
    const recordDataList = await mapWithConcurrencyLimit(validRows, BULK_IO_CONCURRENCY, async (row) => {
        const raw = csvRowToRecordData(row, fields);
        const { recordData } = await datasetRowsService_1.datasetRowsService.normalizeRecordDataForBulkCsvRow({
            datasetId,
            token,
            input: raw,
            datasetCached,
        });
        return recordData;
    });
    const added = await appendDatasetRecords(datasetId, recordDataList);
    return { addedCount: added };
}
async function loadDatasetForBulkAppend(datasetId) {
    const db = await (0, database_1.getDb)();
    const [row] = await db
        .select({
        id: tables_1.trusysDatasetsTable.id,
        type: tables_1.trusysDatasetsTable.type,
        fields: tables_1.trusysDatasetsTable.fields,
        config: tables_1.trusysDatasetsTable.config,
        projectId: tables_1.trusysDatasetsTable.projectId,
        workspaceId: tables_1.trusysDatasetsTable.workspaceId,
    })
        .from(tables_1.trusysDatasetsTable)
        .where((0, drizzle_orm_1.eq)(tables_1.trusysDatasetsTable.id, datasetId))
        .limit(1);
    return row ?? null;
}
//# sourceMappingURL=datasetBulkAppendService.js.map