"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.ddfCsvReader = void 0;
var tslib_1 = require("tslib");
var includes = require("lodash.includes");
var cloneDeep = require("lodash.clonedeep");
var isEmpty = require("lodash.isempty");
var stripBom = require("strip-bom");
var resource_selection_optimizer_1 = require("./resource-selection-optimizer");
var ddfcsv_error_1 = require("./ddfcsv-error");
var ddf_query_validator_1 = require("ddf-query-validator");
var Papa = require("papaparse");
var d3_time_format_1 = require("d3-time-format");
var isValidNumeric = function (val) { return typeof val !== 'number' && !val ? false : true; };
function ddfCsvReader(logger) {
    var internalConcepts = [
        { concept: 'concept', concept_type: 'string', domain: null },
        { concept: 'concept_type', concept_type: 'string', domain: null }
    ];
    var operators = new Map([
        ['$and', function (row, predicates) { return predicates.every(function (p) { return applyFilterRow(row, p); }); }],
        ['$or', function (row, predicates) { return predicates.some(function (p) { return applyFilterRow(row, p); }); }],
        ['$not', function (row, predicate) { return !applyFilterRow(row, predicate); }],
        ['$nor', function (row, predicates) { return !predicates.some(function (p) { return applyFilterRow(row, p); }); }],
        ['$eq', function (rowValue, filterValue) { return rowValue == filterValue; }],
        ['$ne', function (rowValue, filterValue) { return rowValue != filterValue; }],
        ['$gt', function (rowValue, filterValue) { return isValidNumeric(rowValue) && rowValue > filterValue; }],
        ['$gte', function (rowValue, filterValue) { return isValidNumeric(rowValue) && rowValue >= filterValue; }],
        ['$lt', function (rowValue, filterValue) { return isValidNumeric(rowValue) && rowValue < filterValue; }],
        ['$lte', function (rowValue, filterValue) { return isValidNumeric(rowValue) && rowValue <= filterValue; }],
        ['$in', function (rowValue, filterValue) { return filterValue.has(rowValue); }],
        ['$nin', function (rowValue, filterValue) { return !filterValue.has(rowValue); }],
    ]);
    var keyValueLookup = new Map();
    var resourcesLookup = new Map();
    var optimalFilesSet = [];
    var datapackage;
    function loadDataPackage(baseOptions) {
        var datapackagePath = (0, ddf_query_validator_1.getFilePath)(baseOptions.basePath);
        var _a = baseOptions.diagnostic.prepareDiagnosticFor('loadDataPackage'), debug = _a.debug, error = _a.error;
        return new Promise(function (resolve, reject) {
            if (datapackage) {
                return resolve(datapackage);
            }
            baseOptions.fileReader.readText(datapackagePath, function (err, data) {
                if (err) {
                    error('file reading', err);
                    return reject(new ddfcsv_error_1.DdfCsvError(ddfcsv_error_1.FILE_READING_ERROR, err, datapackagePath));
                }
                try {
                    datapackage = JSON.parse(stripBom(data));
                    optimalFilesSet = [];
                    buildResourcesLookup(datapackage);
                    buildKeyValueLookup(datapackage);
                }
                catch (parseErr) {
                    error('json file parsing', parseErr);
                    return reject(new ddfcsv_error_1.DdfCsvError(ddfcsv_error_1.JSON_PARSING_ERROR, parseErr.message, datapackagePath));
                }
                debug('datapackage content is ready');
                resolve(datapackage);
            });
        });
    }
    function loadConcepts(queryParam, options) {
        return tslib_1.__awaiter(this, void 0, void 0, function () {
            var error, conceptQuery, result, concepts, err_1;
            return tslib_1.__generator(this, function (_a) {
                switch (_a.label) {
                    case 0:
                        error = options.diagnostic.prepareDiagnosticFor('loadConcepts').error;
                        setConceptsLookup(internalConcepts, options);
                        conceptQuery = {
                            select: { key: ['concept'], value: ['concept_type', 'domain'] },
                            from: 'concepts'
                        };
                        _a.label = 1;
                    case 1:
                        _a.trys.push([1, 4, , 5]);
                        return [4, queryData(conceptQuery, options)];
                    case 2:
                        concepts = _a.sent();
                        buildConceptsLookup(concepts, options);
                        return [4, reparseConcepts(options)];
                    case 3:
                        result = _a.sent();
                        return [3, 5];
                    case 4:
                        err_1 = _a.sent();
                        error('concepts processing', err_1);
                        throw err_1;
                    case 5: return [2, result];
                }
            });
        });
    }
    function buildConceptsLookup(concepts, options) {
        var entitySetMembershipConcepts = concepts
            .filter(function (concept) { return concept.concept_type === 'entity_set'; })
            .map(function (concept) { return ({
            concept: 'is--' + concept.concept,
            concept_type: 'boolean',
            domain: null
        }); });
        concepts = concepts
            .concat(entitySetMembershipConcepts)
            .concat(internalConcepts);
        setConceptsLookup(concepts, options);
    }
    function reparseConcepts(_a) {
        var conceptsLookup = _a.conceptsLookup;
        var parsingFunctions = new Map([
            ['boolean', function (str) { return str === 'true' || str === 'TRUE'; }],
            ['measure', function (str) { return parseFloat(str); }]
        ]);
        var resources = getResources(['concept']);
        var resourceUpdates = tslib_1.__spreadArray([], tslib_1.__read(resources), false).map(function (resource) {
            return resource.data.then(function (response) {
                var resourceConcepts = Object.keys(response.data[0]);
                var parsingConcepts = new Map();
                resourceConcepts.forEach(function (concept) {
                    var type = conceptsLookup.get(concept).concept_type;
                    var fn = parsingFunctions.get(type);
                    if (fn) {
                        parsingConcepts.set(concept, fn);
                    }
                });
                return response.data.forEach(function (row) {
                    var e_1, _a;
                    try {
                        for (var parsingConcepts_1 = tslib_1.__values(parsingConcepts), parsingConcepts_1_1 = parsingConcepts_1.next(); !parsingConcepts_1_1.done; parsingConcepts_1_1 = parsingConcepts_1.next()) {
                            var _b = tslib_1.__read(parsingConcepts_1_1.value, 2), concept = _b[0], parseFn = _b[1];
                            row[concept] = parseFn(row[concept]);
                        }
                    }
                    catch (e_1_1) { e_1 = { error: e_1_1 }; }
                    finally {
                        try {
                            if (parsingConcepts_1_1 && !parsingConcepts_1_1.done && (_a = parsingConcepts_1.return)) _a.call(parsingConcepts_1);
                        }
                        finally { if (e_1) throw e_1.error; }
                    }
                });
            });
        });
        return Promise.all(resourceUpdates);
    }
    function setConceptsLookup(concepts, options) {
        options.conceptsLookup.clear();
        concepts.forEach(function (row) { return options.conceptsLookup.set(row.concept, row); });
    }
    function query(queryParam, baseOptions) {
        return tslib_1.__awaiter(this, void 0, void 0, function () {
            var _a, warning, error, data, _b, appropriatePlugin, files, err_2;
            return tslib_1.__generator(this, function (_c) {
                switch (_c.label) {
                    case 0:
                        _a = baseOptions.diagnostic.prepareDiagnosticFor('query'), warning = _a.warning, error = _a.error;
                        _c.label = 1;
                    case 1:
                        _c.trys.push([1, 12, , 13]);
                        return [4, (0, ddf_query_validator_1.validateQueryStructure)(queryParam, baseOptions)];
                    case 2:
                        _c.sent();
                        _b = baseOptions;
                        return [4, loadDataPackage(baseOptions)];
                    case 3:
                        _b.datapackage = _c.sent();
                        return [4, loadConcepts(queryParam, baseOptions)];
                    case 4:
                        _c.sent();
                        return [4, (0, ddf_query_validator_1.validateQueryDefinitions)(queryParam, baseOptions)];
                    case 5:
                        _c.sent();
                        if (!(0, ddf_query_validator_1.isSchemaQuery)(queryParam)) return [3, 7];
                        return [4, querySchema(queryParam, baseOptions)];
                    case 6:
                        data = _c.sent();
                        return [3, 11];
                    case 7:
                        appropriatePlugin = (0, resource_selection_optimizer_1.getAppropriatePlugin)(queryParam, baseOptions);
                        if (!appropriatePlugin) return [3, 9];
                        optimalFilesSet = [];
                        return [4, appropriatePlugin.getRecommendedFilesSet()];
                    case 8:
                        files = _c.sent();
                        optimalFilesSet = files;
                        queryParam.optimalFilesSet = [].concat(files, queryParam.optimalFilesSet);
                        warning('get custom optimal files list by a plugin', optimalFilesSet);
                        _c.label = 9;
                    case 9: return [4, queryData(queryParam, baseOptions)];
                    case 10:
                        data = _c.sent();
                        _c.label = 11;
                    case 11: return [3, 13];
                    case 12:
                        err_2 = _c.sent();
                        error('general query error', err_2);
                        throw err_2;
                    case 13: return [2, data];
                }
            });
        });
    }
    function queryData(queryParam, options) {
        var debug = options.diagnostic.prepareDiagnosticFor('queryData').debug;
        var _a = queryParam.select, _b = _a.key, key = _b === void 0 ? [] : _b, _c = _a.value, value = _c === void 0 ? [] : _c, _d = queryParam.from, from = _d === void 0 ? '' : _d, _e = queryParam.where, where = _e === void 0 ? {} : _e, _f = queryParam.join, join = _f === void 0 ? {} : _f, _g = queryParam.order_by, order_by = _g === void 0 ? [] : _g, language = queryParam.language;
        var select = { key: key, value: value };
        debug('start all data loading', queryParam);
        var projection = new Set(select.key.concat(select.value));
        var filterFields = getFilterFields(where).filter(function (field) { return from === 'entities' || !projection.has(field); });
        var resourcesPromise = loadResources(select.key, tslib_1.__spreadArray(tslib_1.__spreadArray([], tslib_1.__read(select.value), false), tslib_1.__read(filterFields), false), language, options, queryParam);
        var joinsPromise = getJoinFilters(join, queryParam, options);
        var entitySetFilterPromise = getEntitySetFilter(select.key, queryParam, options);
        return Promise.all([resourcesPromise, entitySetFilterPromise, joinsPromise])
            .then(function (_a) {
            var _b = tslib_1.__read(_a, 3), resourceResponses = _b[0], entitySetFilter = _b[1], joinFilters = _b[2];
            debug('finish all data loading', queryParam);
            var whereResolved = processWhere(where, joinFilters);
            var filter = mergeFilters(entitySetFilter, whereResolved);
            debug('dataTables processing', queryParam);
            var dataTables = resourceResponses
                .map(function (response) { return processResourceResponse(response, select, filterFields, options); });
            debug('queryResult processing', queryParam);
            var queryResult = joinData.apply(void 0, tslib_1.__spreadArray([select.key, 'overwrite'], tslib_1.__read(dataTables), false)).filter(function (row) { return applyFilterRow(row, filter); })
                .map(function (row) { return fillMissingValues(row, projection); })
                .map(function (row) { return projectRow(row, projection); });
            debug('result ordering', queryParam);
            orderData(queryResult, order_by);
            debug('final result is ready', queryParam);
            return parseTime(queryResult, options);
        });
    }
    function parseTime(result, options) {
        var conceptsLookup = options.conceptsLookup;
        var concepts = Object.keys(result[0] || {});
        var timeConcepts = concepts.map(function (c) { return conceptsLookup.get(c) || {}; }).filter(function (co) { return co.concept_type == 'time'; });
        timeConcepts.forEach(function (_a) {
            var concept = _a.concept;
            var parse = getTimeParser(concept, options);
            result.forEach(function (row) {
                row[concept] = parse(row[concept]);
            });
        });
        return result;
    }
    function getTimeParser(concept, options) {
        var error = options.diagnostic.prepareDiagnosticFor('queryData').error;
        var parsers = {
            year: (0, d3_time_format_1.utcParse)('%Y'),
            month: (0, d3_time_format_1.utcParse)('%Y-%m'),
            day: (0, d3_time_format_1.utcParse)('%Y%m%d'),
            hour: (0, d3_time_format_1.utcParse)('%Y%m%dt%H'),
            minute: (0, d3_time_format_1.utcParse)('%Y%m%dt%H%M'),
            second: (0, d3_time_format_1.utcParse)('%Y%m%dt%H%M%S'),
            week: (0, d3_time_format_1.utcParse)('%Yw%V'),
            quarter: (0, d3_time_format_1.utcParse)('%Yq%q')
        };
        function tryParse(str) {
            for (var i in parsers) {
                var dateObject = parsers[i](str);
                if (dateObject) {
                    return dateObject;
                }
            }
            error('Could not parse time string: ' + str);
            return null;
        }
        if (concept == 'time') {
            return tryParse;
        }
        if (!parsers[concept]) {
            error('No time parser found for time concept: ' + concept);
            return function (str) { return str; };
        }
        return parsers[concept];
    }
    function orderData(data, orderBy) {
        if (orderBy === void 0) { orderBy = []; }
        if (orderBy.length === 0) {
            return;
        }
        var orderNormalized = orderBy.map(function (orderPart) {
            if (typeof orderPart === 'string') {
                return { concept: orderPart, direction: 1 };
            }
            else {
                var concept = Object.keys(orderPart)[0];
                var direction = (orderPart[concept] === 'asc' ? 1 : -1);
                return { concept: concept, direction: direction };
            }
        });
        var n = orderNormalized.length;
        data.sort(function (a, b) {
            for (var i = 0; i < n; i++) {
                var order = orderNormalized[i];
                if (a[order.concept] < b[order.concept]) {
                    return -1 * order.direction;
                }
                else if (a[order.concept] > b[order.concept]) {
                    return 1 * order.direction;
                }
            }
            return 0;
        });
    }
    function processWhere(where, joinFilters) {
        var result = {};
        for (var field in where) {
            var fieldValue = where[field];
            if (includes(['$and', '$or', '$nor'], field)) {
                result[field] = fieldValue.map(function (subFilter) { return processWhere(subFilter, joinFilters); });
            }
            else if (field === '$in' || field === '$nin') {
                result[field] = new Set(fieldValue);
            }
            else if (typeof joinFilters[fieldValue] !== 'undefined') {
                Object.assign(result, joinFilters[fieldValue]);
            }
            else if (typeof fieldValue === 'object') {
                result[field] = processWhere(fieldValue, joinFilters);
            }
            else {
                result[field] = fieldValue;
            }
        }
        return result;
    }
    function mergeFilters() {
        var filters = [];
        for (var _i = 0; _i < arguments.length; _i++) {
            filters[_i] = arguments[_i];
        }
        return filters.reduce(function (a, b) {
            a.$and.push(b);
            return a;
        }, { $and: [] });
    }
    function querySchema(queryParam, baseOptions) {
        var _a = baseOptions.diagnostic.prepareDiagnosticFor('query'), debug = _a.debug, error = _a.error;
        var getSchemaFromCollection = function (collectionPar) {
            debug("get schema for collection ".concat(collectionPar));
            return baseOptions.datapackage.ddfSchema[collectionPar].map(function (_a) {
                var primaryKey = _a.primaryKey, value = _a.value;
                return ({ key: primaryKey, value: value });
            });
        };
        var collection = queryParam.from.split('.')[0];
        if (baseOptions.datapackage.ddfSchema[collection]) {
            return getSchemaFromCollection(collection);
        }
        else if (collection === '*') {
            return Object.keys(baseOptions.datapackage.ddfSchema)
                .map(getSchemaFromCollection)
                .reduce(function (a, b) { return a.concat(b); });
        }
        else {
            var message = "No valid collection (".concat(collection, ") for schema query");
            error(message);
            throwError(new ddfcsv_error_1.DdfCsvError(ddfcsv_error_1.DDF_ERROR, message));
        }
    }
    function fillMissingValues(row, projection) {
        var e_2, _a;
        try {
            for (var projection_1 = tslib_1.__values(projection), projection_1_1 = projection_1.next(); !projection_1_1.done; projection_1_1 = projection_1.next()) {
                var field = projection_1_1.value;
                if (typeof row[field] === 'undefined') {
                    row[field] = null;
                }
            }
        }
        catch (e_2_1) { e_2 = { error: e_2_1 }; }
        finally {
            try {
                if (projection_1_1 && !projection_1_1.done && (_a = projection_1.return)) _a.call(projection_1);
            }
            finally { if (e_2) throw e_2.error; }
        }
        return row;
    }
    function applyFilterRow(row, filter) {
        return Object.keys(filter).every(function (filterKey) {
            var operator = operators.get(filterKey);
            if (operator) {
                return operator(row, filter[filterKey]);
            }
            else if (typeof filter[filterKey] !== 'object') {
                return operators.get('$eq')(row[filterKey], filter[filterKey]);
            }
            else {
                return applyFilterRow(row[filterKey], filter[filterKey]);
            }
        });
    }
    function getJoinFilters(join, queryParam, options) {
        return Promise.all(Object.keys(join).map(function (joinID) { return getJoinFilter(joinID, join[joinID], queryParam, options); }))
            .then(function (results) { return results.reduce(mergeObjects, {}); });
    }
    function mergeObjects(a, b) {
        return Object.assign(a, b);
    }
    function getJoinFilter(joinID, join, queryParam, options) {
        var _a;
        if (options.conceptsLookup.get(join.key).concept_type === 'time') {
            return Promise.resolve((_a = {}, _a[joinID] = join.where, _a));
        }
        else {
            return query({
                select: { key: [join.key] },
                where: join.where,
                from: options.conceptsLookup.has(join.key) ? 'entities' : 'concepts'
            }, Object.assign({ joinID: joinID }, cloneDeep(options)))
                .then(function (result) {
                var _a, _b;
                return (_a = {},
                    _a[joinID] = (_b = {},
                        _b[join.key] = {
                            $in: new Set(result.map(function (row) { return row[join.key]; }))
                        },
                        _b),
                    _a);
            });
        }
    }
    function getFilterFields(filter) {
        var fields = [];
        for (var field in filter) {
            if (includes(['$and', '$or', '$not', '$nor'], field)) {
                filter[field].map(getFilterFields).forEach(function (subFields) { return fields.push.apply(fields, tslib_1.__spreadArray([], tslib_1.__read(subFields), false)); });
            }
            else {
                fields.push(field);
            }
        }
        return tslib_1.__spreadArray([], tslib_1.__read(new Set(fields)), false);
    }
    function filterConceptsByType(conceptTypes, queryKey, options) {
        var e_3, _a;
        var conceptStrings = queryKey || Array.from(options.conceptsLookup.keys());
        var concepts = [];
        try {
            for (var conceptStrings_1 = tslib_1.__values(conceptStrings), conceptStrings_1_1 = conceptStrings_1.next(); !conceptStrings_1_1.done; conceptStrings_1_1 = conceptStrings_1.next()) {
                var conceptString = conceptStrings_1_1.value;
                var concept = options.conceptsLookup.get(conceptString);
                if (includes(conceptTypes, concept.concept_type)) {
                    concepts.push(concept);
                }
            }
        }
        catch (e_3_1) { e_3 = { error: e_3_1 }; }
        finally {
            try {
                if (conceptStrings_1_1 && !conceptStrings_1_1.done && (_a = conceptStrings_1.return)) _a.call(conceptStrings_1);
            }
            finally { if (e_3) throw e_3.error; }
        }
        return concepts;
    }
    function getEntityConceptRenameMap(queryKey, resourceKey, options) {
        var resourceKeySet = new Set(resourceKey);
        var entityConceptTypes = ['entity_set', 'entity_domain'];
        var queryEntityConcepts = filterConceptsByType(entityConceptTypes, queryKey, options);
        if (queryEntityConcepts.length === 0) {
            return new Map();
        }
        var allEntityConcepts = filterConceptsByType(entityConceptTypes, null, options);
        return queryEntityConcepts
            .map(function (concept) { return allEntityConcepts
            .filter(function (lookupConcept) {
            if (concept.concept_type === 'entity_set') {
                return resourceKeySet.has(lookupConcept.concept) &&
                    lookupConcept.concept !== concept.concept &&
                    (lookupConcept.domain === concept.domain ||
                        lookupConcept.concept === concept.domain);
            }
            else {
                return resourceKeySet.has(lookupConcept.concept) &&
                    lookupConcept.concept !== concept.concept &&
                    lookupConcept.domain === concept.concept;
            }
        })
            .reduce(function (map, aliasConcept) { return map.set(aliasConcept.concept, concept.concept); }, new Map()); }).reduce(function (mapA, mapB) { return new Map(tslib_1.__spreadArray(tslib_1.__spreadArray([], tslib_1.__read(mapA), false), tslib_1.__read(mapB), false)); }, new Map());
    }
    function getEntitySetFilter(conceptStrings, queryParam, options) {
        var promises = filterConceptsByType(['entity_set'], conceptStrings, options)
            .map(function (concept) { return query({
            select: { key: [concept.domain], value: ['is--' + concept.concept] },
            from: 'entities'
        }, Object.assign({}, cloneDeep(options)))
            .then(function (result) {
            var _a;
            return (_a = {},
                _a[concept.concept] = {
                    $in: new Set(result
                        .filter(function (row) { return row['is--' + concept.concept]; })
                        .map(function (row) { return row[concept.domain]; }))
                },
                _a);
        }); });
        return Promise.all(promises).then(function (results) {
            return results.reduce(function (a, b) { return Object.assign(a, b); }, {});
        });
    }
    function getResources(key, value) {
        if (!value || value.length === 0 || key[0] === value) {
            return new Set(tslib_1.__spreadArray([], tslib_1.__read(keyValueLookup
                .get(createKeyString(key))
                .values()), false).reduce(function (a, b) { return a.concat(b); }));
        }
        if (Array.isArray(value)) {
            return value
                .map(function (singleValue) { return getResources(key, singleValue); })
                .reduce(function (resultSet, resources) { return new Set(tslib_1.__spreadArray(tslib_1.__spreadArray([], tslib_1.__read(resultSet), false), tslib_1.__read(resources), false)); }, new Set());
        }
        var oneKeyOneValueResourcesArray = keyValueLookup
            .get(createKeyString(key))
            .get(value);
        if (oneKeyOneValueResourcesArray) {
            oneKeyOneValueResourcesArray = oneKeyOneValueResourcesArray
                .filter(function (v) { return isEmpty(optimalFilesSet) || includes(optimalFilesSet, v.path); });
        }
        return new Set(oneKeyOneValueResourcesArray);
    }
    function processResourceResponse(response, select, filterFields, options) {
        var resourcePK = response.resource.schema.primaryKey;
        var resourceProjection = new Set(tslib_1.__spreadArray(tslib_1.__spreadArray(tslib_1.__spreadArray([], tslib_1.__read(resourcePK), false), tslib_1.__read(select.value), false), tslib_1.__read(filterFields), false));
        var renameMap = getEntityConceptRenameMap(select.key, resourcePK, options);
        return response.data
            .map(function (row) { return projectRow(row, resourceProjection); })
            .map(function (row) { return renameHeaderRow(row, renameMap); });
    }
    function loadResources(key, value, language, options, queryParam) {
        var debug = options.diagnostic.prepareDiagnosticFor('loadResource').debug;
        var resources = getResources(key, value);
        debug('resources list by query', { queryParam: queryParam, resources: tslib_1.__spreadArray([], tslib_1.__read(resources), false) });
        return Promise.all(tslib_1.__spreadArray([], tslib_1.__read(resources), false).map(function (resource) { return loadResource(resource, language, options); }));
    }
    function projectRow(row, projectionSet) {
        var e_4, _a;
        var result = {};
        try {
            for (var _b = tslib_1.__values(Object.keys(row)), _c = _b.next(); !_c.done; _c = _b.next()) {
                var concept = _c.value;
                if (projectionSet.has(concept)) {
                    result[concept] = row[concept];
                }
            }
        }
        catch (e_4_1) { e_4 = { error: e_4_1 }; }
        finally {
            try {
                if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
            }
            finally { if (e_4) throw e_4.error; }
        }
        return result;
    }
    function renameHeaderRow(row, renameMap) {
        var e_5, _a;
        var result = {};
        try {
            for (var _b = tslib_1.__values(Object.keys(row)), _c = _b.next(); !_c.done; _c = _b.next()) {
                var concept = _c.value;
                result[renameMap.get(concept) || concept] = row[concept];
            }
        }
        catch (e_5_1) { e_5 = { error: e_5_1 }; }
        finally {
            try {
                if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
            }
            finally { if (e_5) throw e_5.error; }
        }
        return result;
    }
    function joinData(key, joinMode) {
        var data = [];
        for (var _i = 2; _i < arguments.length; _i++) {
            data[_i - 2] = arguments[_i];
        }
        if (data.length === 1) {
            return data[0];
        }
        var canonicalKey = key.slice(0).sort();
        var dataMap = data.reduce(function (result, dataPar) {
            dataPar.forEach(function (row) {
                var keyString = canonicalKey.map(function (concept) { return row[concept]; }).join(',');
                if (result.has(keyString)) {
                    var resultRow = result.get(keyString);
                    joinRow(resultRow, row, joinMode);
                }
                else {
                    result.set(keyString, row);
                }
            });
            return result;
        }, new Map());
        return tslib_1.__spreadArray([], tslib_1.__read(dataMap.values()), false);
    }
    function joinRow(resultRow, sourceRow, mode) {
        switch (mode) {
            case 'overwrite':
                Object.assign(resultRow, sourceRow);
                break;
            case 'translation':
                for (var concept in sourceRow) {
                    if (sourceRow[concept] !== '') {
                        resultRow[concept] = sourceRow[concept];
                    }
                }
                break;
            case 'overwriteWithError':
                for (var concept in sourceRow) {
                    if (resultRow[concept] !== undefined && resultRow[concept] !== sourceRow[concept]) {
                        var sourceRowStr = JSON.stringify(sourceRow);
                        var resultRowStr = JSON.stringify(resultRow);
                        var errStr = "JOIN Error: two resources have different data for \"".concat(concept, "\": ").concat(sourceRowStr, ",").concat(resultRowStr);
                        throwError(new ddfcsv_error_1.DdfCsvError(ddfcsv_error_1.DDF_ERROR, errStr));
                    }
                    else {
                        resultRow[concept] = sourceRow[concept];
                    }
                }
                break;
        }
    }
    function throwError(error) {
        var currentLogger = logger || console;
        currentLogger.error(error.message);
        throw error;
    }
    function createKeyString(key, row) {
        if (row === void 0) { row = false; }
        var canonicalKey = key.slice(0).sort();
        if (!row) {
            return canonicalKey.join(',');
        }
        else {
            return canonicalKey.map(function (concept) { return row[concept]; }).join(',');
        }
    }
    function loadResource(resource, language, options) {
        var warning = options.diagnostic.prepareDiagnosticFor('loadResource').warning;
        var filePromises = [];
        if (typeof resource.data === 'undefined') {
            resource.data = loadFile(resource.path, options);
        }
        filePromises.push(resource.data);
        var languageValid = typeof language !== 'undefined' && includes(getLanguages(options), language);
        var languageLoaded = typeof resource.translations[language] !== 'undefined';
        if (languageValid) {
            if (!languageLoaded) {
                var translationPath_1 = "lang/".concat(language, "/").concat(resource.path);
                resource.translations[language] = loadFile(translationPath_1, options)
                    .catch(function (err) {
                    warning("translation file ".concat(translationPath_1), err);
                    return Promise.resolve({});
                });
            }
            filePromises.push(resource.translations[language]);
        }
        return Promise.all(filePromises).then(function (fileResponses) {
            var filesData = fileResponses.map(function (resp) { return resp.data || []; });
            var primaryKey = resource.schema.primaryKey;
            var data = joinData.apply(void 0, tslib_1.__spreadArray([primaryKey, 'translation'], tslib_1.__read(filesData), false));
            return { data: data, resource: resource };
        });
    }
    function getLanguages(options) {
        if (!options.datapackage.translations) {
            return [];
        }
        return options.datapackage.translations.map(function (lang) { return lang.id; });
    }
    function loadFile(filePath, options) {
        var _a = options.diagnostic.prepareDiagnosticFor('loadFile'), debug = _a.debug, error = _a.error;
        var fullFilePath = (0, ddf_query_validator_1.getFilePath)(options.basePath, filePath);
        debug("start reading \"".concat(filePath, "\""));
        return new Promise(function (resolve, reject) {
            options.fileReader.readText(fullFilePath, function (err, data) {
                if (err) {
                    error("fail \"".concat(filePath, "\" reading"), err);
                    return reject(new ddfcsv_error_1.DdfCsvError(ddfcsv_error_1.FILE_READING_ERROR, err, fullFilePath));
                }
                Papa.parse(stripBom(data), {
                    header: true,
                    skipEmptyLines: true,
                    dynamicTyping: true,
                    complete: function (result) {
                        debug("finish reading \"".concat(filePath, "\""));
                        resolve(result);
                    },
                    error: function (parseErr) {
                        error("fail \"".concat(filePath, "\" parsing"), parseErr);
                        reject(new ddfcsv_error_1.DdfCsvError(ddfcsv_error_1.CSV_PARSING_ERROR, parseErr, filePath));
                    }
                });
            });
        });
    }
    function buildResourcesLookup(datapackagePar) {
        if (resourcesLookup.size > 0) {
            return resourcesLookup;
        }
        datapackagePar.resources.forEach(function (resource) {
            if (!Array.isArray(resource.schema.primaryKey)) {
                resource.schema.primaryKey = [resource.schema.primaryKey];
            }
            resource.translations = {};
            resourcesLookup.set(resource.name, resource);
        });
        return resourcesLookup;
    }
    function buildKeyValueLookup(datapackagePar) {
        if (keyValueLookup.size > 0) {
            return keyValueLookup;
        }
        for (var collection in datapackagePar.ddfSchema) {
            datapackagePar.ddfSchema[collection].map(function (kvPair) {
                var key = createKeyString(kvPair.primaryKey);
                var resources = kvPair.resources.map(function (resourceName) { return resourcesLookup.get(resourceName); });
                if (keyValueLookup.has(key)) {
                    keyValueLookup.get(key).set(kvPair.value, resources);
                }
                else {
                    keyValueLookup.set(key, new Map([[kvPair.value, resources]]));
                }
            });
        }
        return keyValueLookup;
    }
    return {
        query: query
    };
}
exports.ddfCsvReader = ddfCsvReader;
//# sourceMappingURL=ddf-csv.js.map