// import { ml, } from './ml';
import { array as ArrayStat, } from 'ml-stat';
import { util as utils, } from './util';

const transformConfigMap = {
  scale: 'scaleOptions',
  descale: 'descaleOptions',
  label: 'labelOptions',
  labelEncoder: 'labelOptions',
  labeldecode: 'labelOptions',
  labelDecode: 'labelOptions',
  labelDecoder: 'labelOptions',
  onehot: 'oneHotOptions',
  oneHot: 'oneHotOptions',
  oneHotEncode: 'oneHotOptions',
  oneHotEncoder: 'oneHotOptions',
  reducer: 'reducerOptions',
  reduce: 'reducerOptions',
  merge: 'mergeData',
};

export type DataSetTransform = {
  [index: string]: string[]|any;
}
export type FitColumnsOption = {
  name: string;
  options?: {
    [index: string]: any;
    // strategy?: string;
  }
};
export type FitColumnsOptions = FitColumnsOption[];
export type Vector = number[];
export type Matrix = number[][];
export type ReverseColumnMatrixOptions = {
  vectors?: Matrix;
  labels?: string[];
};
export type ReverseColumnVectorOptions = {
  vector?: Vector;
  labels?: string[];
};
export type ColumnArrayOptions = {
  [index: string]: any;
}
export type variableLabels = Array<Array<string>>;
export type OneHotEncoderOptions = {
  merge: boolean;
  data: Vector;
  columnArrayOptions: ColumnArrayOptions;
  prefix: string;
  // [index: string]: any;
}
export type OneHotEncodedData = {
  [index: string]: Vector;
}

export type Datum = {
  [index: string]: any;
};
export type Data = Datum[];

/**
 * class for manipulating an array of objects, typically from CSV data
 * @class DataSet
 * @memberOf preprocessing
 */
export class DataSet  {
  config : { [index: string]: any;};
  data : Data;
  labels : any;
  encoders : any;
  scalers : any;
  selectColumns :(...args:any[])=>any;
  columnArray : (...args:any[])=>any;
  encodeObject : (...args:any[])=>any;
  oneHotEncoder : (...args:any[])=>any;
  oneHotDecoder : (...args:any[])=>any;
  columnMatrix : (...args:any[])=>any;
  reverseColumnMatrix : (...args:any[])=>any;
  reverseColumnVector : (...args:any[])=>any;
  getTransforms : (...args:any[])=>any;
  static encoders: any;
  static data: any;

  /**
   * Allows for fit transform short hand notation
   * @example
DataSet.getTransforms({
  Age: ['scale',],
  Rating: ['label',],  }); //=> [
//   {
//    name: 'Age', options: { strategy: 'scale', }, },
//   },
//   { 
//    name: 'Rating', options: { strategy: 'label', }, 
//   },
// ];
   * @param {Object} transforms 
   * @returns {Array<Object>} returns fit columns, columns property
   */
  static getTransforms(transforms:DataSetTransform = {}):FitColumnsOptions {
    return Object.keys(transforms).reduce((result:FitColumnsOptions, columnName) => {
      const transformColumnObject = transforms[ columnName ];
      const transformObject = {
        name: columnName,
        options: {
          strategy: (Array.isArray(transformColumnObject))
            ? transformColumnObject[ 0 ]
            : transformColumnObject,
        },
      };
      if (Array.isArray(transformColumnObject) && transformColumnObject.length > 1) {
        //@ts-ignore
        transformObject.options[ transformConfigMap[ transformColumnObject[ 0 ] ] ] = transformColumnObject[ 1 ];
      }
      result.push(transformObject);
      return result;
    }, []);
  }
  /**
   * returns an array of objects by applying labels to matrix of columns
   * @example
const data = [{ Age: '44', Salary: '44' },
{ Age: '27', Salary: '27' }]
const AgeDataSet = new MS.DataSet(data);
const dependentVariables = [ [ 'Age', ], [ 'Salary', ], ];
const AgeSalMatrix = AgeDataSet.columnMatrix(dependentVariables); // =>
//  [ [ '44', '72000' ],
//  [ '27', '48000' ] ];
MS.DataSet.reverseColumnMatrix({vectors:AgeSalMatrix,labels:dependentVariables}); // => [{ Age: '44', Salary: '44' },
{ Age: '27', Salary: '27' }]
   * 
   * @param {*} options 
   * @param {Array[]} options.vectors - array of vectors
   * @param {String[]} options.labels - array of labels
   * @returns {Object[]} an array of objects with properties derived from options.labels
   */
  static reverseColumnMatrix(options:ReverseColumnMatrixOptions = {}):Data {
    const { vectors=[], labels=[], } = options;
    const features = (Array.isArray(labels) && Array.isArray(labels[ 0 ]))
      ? labels
      : labels.map(label => [label, ]);
    return vectors.reduce((result:Data, val:Vector) => {
      result.push(val.reduce((prop:Datum, value, index) => {
        prop[ features[ index ][ 0 ] ] = val[ index ];
        return prop;
      }, {}));
      return result;
    }, []);
  }
  static reverseColumnVector(options:ReverseColumnVectorOptions = {}):Data {
    const { vector=[], labels=[], } = options;
    const features = (Array.isArray(labels) && Array.isArray(labels[ 0 ]))
      ? labels
      : labels.map(label => [label, ]);
    return vector.reduce((result:Data, val) => {
      result.push(
        { [ features[ 0 ][ 0 ] ]: val, }
      );
      return result;
    }, []);
  }
  /**
   * Returns an object into an one hot encoded object
   * @example
const labels = ['apple', 'orange', 'banana',];
const prefix = 'fruit_';
const name = 'fruit';
const options = { labels, prefix, name, };
const data = {
  fruit: 'apple',
};
EncodedCSVDataSet.encodeObject(data, options); // => { fruit_apple: 1, fruit_orange: 0, fruit_banana: 0, }
   * @param {Object} data - object to encode 
   * @param {{labels:Array<String>,prefix:String,name:String}} options - encoded object options
   * @returns {Object} one hot encoded object
   */
  static encodeObject(data: Datum, options: { labels: string[]; prefix: string; name: string;}):Datum {
    const { labels, prefix, name, } = options;
    const encodedData = labels.reduce((encodedObj:Datum, label) => {
      const oneHotLabelArrayName = `${prefix}${label}`;
      encodedObj[ oneHotLabelArrayName ] = (label && data[ name ] && data[ name ].toString() === label.toString()) ? 1 : 0;
      return encodedObj;
    }, {});
    return encodedData;
  }
  /**
 * returns a new object of one hot encoded values
 * @example
 * // [ 'Brazil','Mexico','Ghana','Mexico','Ghana','Brazil','Mexico','Brazil','Ghana', 'Brazil' ]
const originalCountry = dataset.columnArray('Country'); 

// { originalCountry:
//    { Country_Brazil: [ 1, 0, 0, 0, 0, 1, 0, 1, 0, 1 ],
//      Country_Mexico: [ 0, 1, 0, 1, 0, 0, 1, 0, 0, 0 ],
//      Country_Ghana: [ 0, 0, 1, 0, 1, 0, 0, 0, 1, 0 ] },
//     }
const oneHotCountryColumn = dataset.oneHotEncoder('Country'); 
  * @param {string} name - csv column header, or JSON object property name 
  * @param options 
  * @see {@link http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.OneHotEncoder.html}
  * @return {Object}
  */
  static oneHotEncoder(this: any, name:string, options:OneHotEncoderOptions):OneHotEncodedData {
    const config = Object.assign({
      merge: true,
    }, options);
    const labelData = config.data || this.columnArray(name, config.columnArrayOptions);
    const labels = Array.from(new Set(labelData).values());
    const prefix = config.prefix || `${name}_`;
    const encodedData = labelData.reduce(
      (result:OneHotEncodedData, val, index, arr) => {
        labels.forEach(encodedLabel => {
          const oneHotLabelArrayName = `${prefix}${encodedLabel}`;
          const oneHotVal = (val === encodedLabel) ? 1 : 0;
          if (Array.isArray(result[ oneHotLabelArrayName ])) {
            result[ oneHotLabelArrayName ].push(oneHotVal);
          } else {
            result[ oneHotLabelArrayName ] = [oneHotVal, ];
          }
        });
        return result;
      }, {});
    if (this.encoders.has(name) && config.merge) {
      this.encoders.get(name).labels = Array.from(new Set(labels.concat(this.encoders.get(name).labels)));
      // this.encoders.get(name);
    } else {
      this.encoders.set(name, {
        name,
        labels,
        prefix,
      });
    }
    return encodedData;
  }
  /**
   * Return one hot encoded data
   * @example
const csvData = [{
    'Country': 'Brazil',
    'Age': '44',
    'Salary': '72000',
    'Purchased': 'N',
  },
  {
    'Country': 'Mexico',
    'Age': '27',
    'Salary': '48000',
    'Purchased': 'Yes',
  },
  ...
];
const EncodedCSVDataSet = new ms.preprocessing.DataSet(csvData);
EncodedCSVDataSet.fitColumns({
  columns: [
    {
      name: 'Country',
      options: { strategy: 'onehot', },
    },
  ],
});

EncodedCSVDataSet.oneHotDecoder('Country);// =>
// [ { Country: 'Brazil' },
//  { Country: 'Mexico' },
//  { Country: 'Ghana' },
//  { Country: 'Mexico' },
//   ...]
   * @param {string} name - column name 
   * @param options 
   * @returns {Array<Object>} returns an array of objects from an one hot encoded column
   */
  static oneHotDecoder(name: any, options: any) {
    const config = Object.assign({
      // handle_unknown: 'error'
    }, options);
    const encoderMap = config.encoders || this.encoders;
    const prefix = config.prefix || encoderMap.get(name).prefix;
    const labels = config.labels || encoderMap.get(name).labels;
    const encodedData = config.data || this.oneHotColumnArray(name, config.oneHotColumnArrayOptions);
    // console.log({ encodedData, encoderMap, prefix });
    return encodedData.reduce((result: { [x: number]: string; }[], val: { [x: string]: number; }) => {
      const columnNames = Object.keys(val).filter(prop => val[ prop ] === 1 && (labels.indexOf(prop.replace(prefix, '')) !== -1 || labels.map((label: any) => String(label)).indexOf(prop.replace(prefix, '')) !== -1));
      const columnName = columnNames[ 0 ] || '';
      // console.log({ columnName, columnNames, labels, val},Object.keys(val));
      const datum = {
        [ name ]: columnName.replace(prefix, ''),
      };
      result.push(datum);
      return result;
    }, []);
  }
  static oneHotColumnArray(name: any, oneHotColumnArrayOptions: any): any {
    throw new Error("Method not implemented.");
  }
  /**
   * returns a list of objects with only selected columns as properties
 * @example
const data = [{ Age: '44', Salary: '44' , Height: '34' },
{ Age: '27', Salary: '44' , Height: '50'  }]
const AgeDataSet = new MS.DataSet(data);
const cols = [ 'Age', 'Salary' ];
const selectedCols = CSVDataSet.selectColumns(cols); // => [{ Age: '44', Salary: '44' },
{ Age: '27', Salary: '27' }]
   * 
   * @param {String[]} names - array of selected columns
   * @param {*} options 
   * @returns {Object[]} an array of objects with properties derived from names
   */
  static selectColumns(names: any[], options:any = {}) {
    const config = Object.assign({}, options);
    const data = config.data || this.data;
    return data.reduce((result: {}[], val: { [x: string]: any; }) => {
      const selectedData:any = {};
      names.forEach((name: string | number) => {
        selectedData[ name ] = val[ name ];
      });
      result.push(selectedData);
      return result;
    }, []);
  }
  /**
   * returns a new array of a selected column from an array of objects, can filter, scale and replace values
   * @example 
   * //column Array returns column of data by name
// [ '44','27','30','38','40','35','','48','50', '37' ]
const OringalAgeColumn = dataset.columnArray('Age'); 
  * @param {string} name - csv column header, or JSON object property name 
  * @param options 
  * @param {function} [options.prefilter=(arr[val])=>true] - prefilter values to return
  * @param {function} [options.filter=(arr[val])=>true] - filter values to return
  * @param {function} [options.replace.test=undefined] - test function for replacing values (arr[val])
  * @param {(string|number|function)} [options.replace.value=undefined] - value to replace (arr[val]) if replace test is true, if a function (result,val,index,arr,name)=>your custom value
  * @param {number} [options.parseIntBase=10] - radix value for parseInt
  * @param {boolean} [options.parseFloat=false] - convert values to floats 
  * @param {boolean} [options.parseInt=false] - converts values to ints 
  * @param {boolean} [options.scale=false] - standard or minmax feature scale values 
  * @returns {array}
  */
  static columnArray(name: string | number, options:any = {}) {
    const config = Object.assign({
      prefilter: () => true,
      filter: () => true,
      replace: {
        test: undefined,
        value: undefined,
      },
      parseInt: false,
      parseIntBase: 10,
      parseFloat: (options.scale) ? true : false,
      scale: false,
    }, options);
    const data = config.data || this.data;
    const modifiedColumn = data
      .filter(config.prefilter)
      .reduce((result: number[], val: { [x: string]: any; }, index: any, arr: any) => {
        let objVal = val[ name ];
        let returnVal = (typeof config.replace.test === 'function') ?
          config.replace.test(objVal) ?
            typeof config.replace.value === 'function' ?
              config.replace.value(result, val, index, arr, name) :
              config.replace.value :
            objVal :
          objVal;
        if (config.filter(returnVal)) {
          if (config.parseInt) result.push(parseInt(returnVal, config.parseIntBase));
          else if (config.parseFloat) result.push(parseFloat(returnVal));
          else result.push(returnVal);
        }
        return result;
      }, []);
    if (typeof config.scale === 'function') {
      return modifiedColumn.map(config.scale);
    } else if (config.scale) {
      switch (config.scale) {
      case 'standard':
        return utils.StandardScaler(modifiedColumn);
      case 'log':
        return utils.LogScaler(modifiedColumn);
      case 'exp':
        return utils.ExpScaler(modifiedColumn);
      case 'normalize':
      default:
        return utils.MinMaxScaler(modifiedColumn);
      }
    } else {
      return modifiedColumn;
    }
  }
  
  /**
   * returns a matrix of values by combining column arrays into a matrix
   * @example const csvObj = new DataSet([{col1:1,col2:5},{col1:2,col2:6}]);
csvObj.columnMatrix([['col1',{parseInt:true}],['col2']]); // =>
//[ 
//  [1,5], 
//  [2,6], 
//]
  * @param {Array} [vectors=[]] - array of arguments for columnArray to merge columns into a matrix
  * @param {Array} [data=[]] - array of data to convert to matrix
  * @returns {Array} a matrix of column values 
  */
  static columnMatrix(vectors = [], data = []) {
    const options = (data.length) ? { data, } : {};
    const columnVectors = (Array.isArray(vectors) && Array.isArray(vectors[ 0 ]))
      ? vectors
      : vectors.map(vector => [vector, options, ]);
    const vectorArrays = columnVectors
      //@ts-ignore
      .map((vec:any[]) => DataSet.columnArray.call(this, ...vec));
        
    return utils.pivotArrays(vectorArrays);
  }
  /**
   * returns a JavaScript Object from a Map (supports nested Map Objects)
   * @example const csvObj = new DataSet([{col1:1,col2:5},{col1:2,col2:6}]);
csvObj.columnMatrix([['col1',{parseInt:true}],['col2']]); // =>
//[ 
//  [1,5], 
//  [2,6], 
//]
  * @param {Map} mapObj - Map to convert into JavaScript Object
  * @returns {Object} JavaScript Object converted from a Map
  */
  static mapToObject(mapObj = new Map()){
    return Array.from(mapObj.keys())
      .reduce((result, val) => {
        const mapVal = mapObj.get(val);
        if (mapVal instanceof Map) {
          result[ val ] = DataSet.mapToObject(mapVal);
        } else if (typeof mapVal === 'function') {
          result[ val ] = `[Function ${mapVal.name}]`;
        } else {
          result[ val ] = JSON.parse(JSON.stringify(mapVal));
        }
        return result;
      }, {});
  }
  /**
   * returns 0 or 1 depending on the input value
   * @example DataSet.getBinaryValue('true') // => 1
DataSet.getBinaryValue('false') // => 0
DataSet.getBinaryValue('No') // => 0
DataSet.getBinaryValue(false) // => 0
  * @param {String|Number} [value=''] - value to convert to a 1 or a 0
  * @returns {Number} 0 or 1 depending on truthiness of value
  */
  static getBinaryValue(value:string|boolean='') {
    if (!value) return 0;
    switch (value) {
    case false:
    case 'N':
    case 'n':
    case 'NO':
    case 'No':
    case 'no':
    case 'False':
    case 'F':
    case 'f':
      return 0;
    default:
      return 1;
    }
  }
  /**
   * creates a new raw data instance for preprocessing data for machine learning
   * @example
   * const dataset = new ms.DataSet(csvData);
   * @param {Object[]} dataset
   * @returns {this} 
   */
  constructor(data:Data = [], options = {}) {
    this.config = Object.assign({
      debug: true,
    }, options);
    this.data = [...data, ];
    this.labels = new Map();
    this.encoders = new Map();
    this.scalers = new Map();
    this.selectColumns = DataSet.selectColumns;
    this.columnArray = DataSet.columnArray;
    this.encodeObject = DataSet.encodeObject;
    this.oneHotEncoder = DataSet.oneHotEncoder;
    this.oneHotDecoder = DataSet.oneHotDecoder;
    this.columnMatrix = DataSet.columnMatrix;
    this.reverseColumnMatrix = DataSet.reverseColumnMatrix;
    this.reverseColumnVector = DataSet.reverseColumnVector;
    this.getTransforms = DataSet.getTransforms;
    if (this.config.labels || this.config.encoders || this.config.scalers) {
      this.importFeatures({
        labels: this.config.labels,
        encoders: this.config.encoders,
        scalers: this.config.scalers,
      });
    }
    return this;
  }
  /**
   * returns Object of all encoders and scalers 
   * @example const csvObj = new DataSet([{col1:1,col2:5},{col1:false,col2:6}]);
DataSet.fitColumns({col1:['label',{binary:true}]}); 
Dataset.data // => [{col1:true,col2:5},{col1:false,col2:6}]
Dataset.exportFeatures() //=> { labels: { col1: { "0": false, "1": true, "N": 0, "Yes": 1, "No": 0, "f": 0, "false": 1, } } }
  * @param {Function} [filter=()=>true] - filter function
  * @returns {{labels:Map,encoders:Map,scalers:map}} JavaScript Object of transforms encoders and scalers(labels, encoders, scalers) 
  */
  exportFeatures(options = {}) {
    const config = Object.assign({
    }, options);
    return {
      encoders: DataSet.mapToObject(this.encoders),
      labels: DataSet.mapToObject(this.labels),
      scalers: DataSet.mapToObject(this.scalers),
    };
  }
  /**
   * set encoders, labels and scalers 
   * @example const csvObj = new DataSet([{col1:1,col2:5},{col1:false,col2:6}]);
DataSet.fitColumns({col1:['label',{binary:true}]}); 
Dataset.data // => [{col1:true,col2:5},{col1:false,col2:6}]
Dataset.exportFeatures() //=> { labels: { col1: { "0": false, "1": true, "N": 0, "Yes": 1, "No": 0, "f": 0, "false": 1, } } }
  * @param {{labels:Map,encoders:Map,scalers:map}} [features={}] - JavaScript Object of transforms encoders and scalers(labels, encoders, scalers) 
  */
  importFeatures(features:any = {}) {
    Object.keys(features.encoders || {}).forEach(encoderName => { 
      const encoder = features.encoders[ encoderName ];
      this.encoders.set(encoderName, encoder);
    });
    Object.keys(features.labels || {}).forEach(labelName => {
      const labelData = features.labels[labelName];
      const labels:any = Object.keys(labelData)
        .map(labelProp => [labelProp, labelData[ labelProp ], ]);
      if (typeof labelData[ '0' ] !== 'undefined' && typeof labelData[ '1' ] !== 'undefined') {
        labels.push(...[
          [ 0, false, ],
          [ false, 0, ],
          [ null, 0, ],
          [ 'no', 0, ],
          [ 'No', 0, ],
          [ 'NO', 0, ],
          [ 'F', 0, ],
          [ 'f', 0, ],
          [ 'null', 0, ],
          [ '', 0, ],
          [ undefined, 0, ],
          [ 'undefined', 0, ],
          [ 1, true, ],
          [ true, 1, ],
        ]);
      }
      this.labels.set(labelName, new Map(labels));
    });
    Object.keys(features.scalers || {}).forEach(scalerName => {
      let transforms;
      const scalerData = features.scalers[ scalerName ];
      const { config, } = scalerData;
      switch (config.strategy) {
      case 'standard':
        transforms = utils.StandardScalerTransforms(...[undefined, config.nan_value, config.return_nan, scalerData.components,]);
        scalerData.scale = transforms.scale;  
        scalerData.descale = transforms.descale;  
        break;
      case 'normalize':
      case 'minmax':
        transforms = utils.MinMaxScalerTransforms(...[undefined, config.nan_value, config.return_nan, scalerData.components,]);
        scalerData.scale = transforms.scale;  
        scalerData.descale = transforms.descale;   
        break;
      case 'log':
      default:
        scalerData.scale = Math.log;  
        scalerData.descale = Math.exp;  
        break;
      }
      this.scalers.set(scalerName, scalerData);
    });
  }
  /**
   * returns filtered rows of data 
   * @example const csvObj = new DataSet([{col1:1,col2:5},{col1:2,col2:6}]);
csvObj.filterColumn((row)=>row.col1>=2); // =>
//[ 
//  [2,6], 
//]
  * @param {Function} [filter=()=>true] - filter function
  * @returns {Array} filtered array of data 
  */
  filterColumn(filter = () => true) {
    return this.data.filter(filter);
  }
  /**
   * Returns a new array of scaled values which can be reverse (descaled). The scaling transformations are stored on the DataSet
   * @example
//dataset.columnArray('Age') => [ '44','27','30','38','40','35',38.77777777777778,'48','50','37' ]
dataset.columnScale('Age',{strategy:'log'}) // => [ 3.784189633918261,
  3.295836866004329, 3.4011973816621555, 3.6375861597263857, 3.6888794541139363, 3.5553480614894135, 3.657847344866208, 3.8712010109078907, 3.912023005428146, 3.6109179126442243 ]
dataset.scalers.get('Age').scale(45) // => 3.8066624897703196
dataset.scalers.get('Age').descale(3.8066624897703196) // => 45
//this supports, log/exponent, minmax/normalization and standardscaling
   * @param {string} name - name - csv column header, or JSON object property name 
   * @param {string} [options.strategy="log"] - strategy for scaling values 
   * @returns {number[]} returns an array of scaled values
   */
  columnScale(name: any, options = {}) {
    const input = (typeof options === 'string')
      ? { strategy: options, }
      : options;
    const config:any = Object.assign({
      strategy: 'log',
      forced_coercion: false,
    }, input);
    let scaleData = config.data || this.columnArray(name, config.columnArrayOptions);
    let scaledData;
    let transforms;
      
    scaleData = scaleData.filter((datum: any) => typeof datum !== 'undefined')
      .map((datum: any, i: any) => {
        if (typeof datum !== 'number') {
          if (this.config.debug && config.forced_coercion===false) {
            console.error(TypeError(`Each value must be a number, error at index [${name}][${i}]: <${typeof datum}>${datum}`));
          }
          const num = Number(datum);
          if (isNaN(num) && config.forced_coercion) {
            return 0;
          } else if (isNaN(num)) {
            throw TypeError(`Only numerical values in (${name}) can be scaled i: ${i} datum: ${datum}`);
          }
          return num;
        } else return datum;
      });
    switch (config.strategy) {
    case 'standard':
      transforms = utils.StandardScalerTransforms(...[scaleData, config.nan_value, config.return_nan,]);
      this.scalers.set(name, {
        name,
        scale: transforms.scale,
        descale: transforms.descale,
        components: transforms.components,
      });
      scaledData = transforms.values;
      break;
    case 'normalize':
    case 'minmax':
      transforms = utils.MinMaxScalerTransforms(...[scaleData, config.nan_value, config.return_nan,]);     
      this.scalers.set(name, {
        name,
        scale: transforms.scale,
        descale: transforms.descale,
        components: transforms.components,
      });
      scaledData = transforms.values;
      break;
    case 'log':
    default:
      this.scalers.set(name, {
        name,
        scale: Math.log,
        descale: Math.exp,
        components: {
          average : utils.avg(scaleData),
          standard_dev : utils.sd(scaleData),
          maximum : utils.max(scaleData),
          minimum : utils.min(scaleData),
        },
      });
      scaledData = utils.LogScaler(scaleData);
      break;
    }
    this.scalers.get(name).config = config;
    return scaledData;
  }
  /**
   * Returns a new array of descaled values
   * @example
//dataset.columnArray('Age') => [ '44','27','30','38','40','35',38.77777777777778,'48','50','37' ]
const scaledData = [ 3.784189633918261,
  3.295836866004329, 3.4011973816621555, 3.6375861597263857, 3.6888794541139363, 3.5553480614894135, 3.657847344866208, 3.8712010109078907, 3.912023005428146, 3.6109179126442243 ]
dataset.columnDescale('Age') // => [ '44','27','30','38','40','35',38.77777777777778,'48','50','37' ]
   * @param {string} name - name - csv column header, or JSON object property name 
   * @param {string} [options.strategy="log"] - strategy for scaling values 
   * @returns {number[]} returns an array of scaled values
   */
  columnDescale(name: any, options: any) {
    const config = Object.assign({ }, options);
    const scaledData = config.data || this.columnArray(name, config.columnArrayOptions);
    const descaleFunction = this.scalers.get(name).descale;
    return scaledData.map(descaleFunction);
  }
  /**
   * returns a new array and label encodes a selected column
   * @example
   * const oneHotCountryColumn = dataset.oneHotEncoder('Country'); 

// [ 'N', 'Yes', 'No', 'f', 'Yes', 'Yes', 'false', 'Yes', 'No', 'Yes' ] 
const originalPurchasedColumn = dataset.labelEncoder('Purchased');
// [ 0, 1, 0, 0, 1, 1, 1, 1, 0, 1 ]
const encodedBinaryPurchasedColumn = dataset.labelEncoder('Purchased',{ binary:true });
// [ 0, 1, 2, 3, 1, 1, 4, 1, 2, 1 ]
const encodedPurchasedColumn = dataset.labelEncoder('Purchased'); 
  * @param {string} name - csv column header, or JSON object property name 
  * @param options
  * @param {boolean} [options.binary=false] - only replace with (0,1) with binary values 
  * @param {function} options.sortFunction - custom label encoding value sort function
  * @see {@link http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.LabelEncoder.html} 
  * @returns {array}
  */
  labelEncoder(name: any, options: {}) {
    const config:any = Object.assign({
      binary: false,
      merge: true,
    }, options);
    const labelData = config.data || this.columnArray(name, config.columnArrayOptions);
    let labelDataUniqueValues:any = Array.from(new Set(labelData).values()).sort(config.sortFunction);
    const labels = new Map(
      labelDataUniqueValues
        .reduce((result:any, val:any, i:number) => {
          if (config.binary) {
            if (i === 0) {
              result.push(...[
                [0, false,],
                ['0', false,],
                [1, true,],
                ['1', true,],
              ]);
            }
            result.push([val, DataSet.getBinaryValue(val),]);
          } else {
            result.push([val, i,]);
            result.push([i, val,]);
          }
          return result;
        }, [])
    );
    if (this.labels.has(name) && config.merge) {
      this.labels.set(name, new Map([...this.labels.get(name), ...labels,]));
    } else this.labels.set(name, labels);
    const labeledData = (config.binary) ?
      labelData.map(DataSet.getBinaryValue) :
      labelData.map((label: unknown) => labels.get(label));
    return labeledData;
  }
  /**
     * returns a new array and decodes an encoded column back to the original array values
     * @param {string} name - csv column header, or JSON object property name 
     * @param options
     * @returns {array}
     */
  labelDecode(name: any, options:any= {}) {
    const config = Object.assign({}, options);
    const labelData = config.data || this.columnArray(name, config.columnArrayOptions);
    return labelData.map((val: any) => this.labels.get(name).get(val));
  }
  /**
   * Return one hot encoded data
   * @example
const csvData = [{
    'Country': 'Brazil',
    'Age': '44',
    'Salary': '72000',
    'Purchased': 'N',
  },
  {
    'Country': 'Mexico',
    'Age': '27',
    'Salary': '48000',
    'Purchased': 'Yes',
  },
  ...
];
const EncodedCSVDataSet = new ms.preprocessing.DataSet(csvData);
EncodedCSVDataSet.fitColumns({
  columns: [
    {
      name: 'Country',
      options: { strategy: 'onehot', },
    },
  ],
});

EncodedCSVDataSet.oneHotColumnArray('Country);// =>
// [ { Country_Brazil: 1, Country_Mexico: 0, Country_Ghana: 0 },
//   { Country_Brazil: 0, Country_Mexico: 1, Country_Ghana: 0 },
//   { Country_Brazil: 0, Country_Mexico: 0, Country_Ghana: 1 },
//   ...]
   * @param {string} name - column name 
   * @param options 
   * @returns {Array<Object>} returns an array of objects from an one hot encoded column
   */
  oneHotColumnArray(name: any, options: any) {
    const config = Object.assign({
      // handle_unknown: 'error'
    }, options);
    const labels = config.labels || this.encoders.get(name).labels;
    const prefix = config.prefix || this.encoders.get(name).prefix;
    return this.selectColumns(labels.map((label: any) => `${prefix}${label}`));
  }
  /**
 * it returns a new column that reduces a column into a new column object, this is used in data prep to create new calculated columns for aggregrate statistics
 * @example 
const reducer = (result, value, index, arr) => {
result.push(value * 2);
return result;
};
CSVDataSet.columnReducer('DoubleAge', {
columnName: 'Age',
reducer,
}); //=> { DoubleAge: [ 88, 54, 60, 76, 80, 70, 0, 96, 100, 74 ] }
  * @param {String} name - name of new Column 
  * @param {Object} options 
  * @param {String} options.columnName - name property for columnArray selection 
  * @param {Object} options.columnOptions - options property for columnArray  
  * @param {Function} options.reducer - reducer function to reduce into new array, it should push values into the resulting array  
  * @returns {Object} a new object that has reduced array as the value
  */
  columnReducer(name: any, options: { columnName: any; columnOptions: any; reducer: any; }) {
    const newColumn = {
      [ name ]: this.columnArray(options.columnName, options.columnOptions).reduce(options.reducer, []),
    };
    return newColumn;
  }
  /**
   * it returns a new column that is merged onto the data set
   * @example 
CSVDataSet.columnMerge('DoubleAge', [ 88, 54, 60, 76, 80, 70, 0, 96, 100, 74 ]); //=> { DoubleAge: [ 88, 54, 60, 76, 80, 70, 0, 96, 100, 74 ] }
    * @param {String} name - name of new Column 
    * @param {Array} data - new dataset data  
    * @returns {Object} 
    */
  columnMerge(name: any, data=[]) {
    if (this.data.length !== data.length) throw new RangeError(`Merged data column must have the same length(${data.length}) as the DataSet's length (${this.data.length})`);
    return {
      [name]: data,
    };
  }
  /**
   * Inverses transform on an object
   * @example
DataSet.data; //[{
//   Age: 0.6387122698222066,
//   Salary: 72000,
//   Purchased: 0,
//   Country_Brazil: 1,
//   Country_Mexico: 0,
//   Country_Ghana: 0,
// }, ...] 
DataSet.inverseTransformObject(DataSet.data[0]); // => {
//  Country: 'Brazil', 
//  Age: 44, 
//  Salary: 72000, 
//  Purchased: 'N', 
// };
   * @param data 
   * @param options 
   * @returns {Object} returns object with inverse transformed data
   */
  inverseTransformObject(data: { [x: string]: any; }, options: {}) {
    const config = Object.assign({
      removeValues: false,
    }, options);
    const removedColumns: string[] = [];
    let transformedObject = Object.assign({}, data);
    const columnNames = Object.keys(this.data[ 0 ]);
    const scaledData = columnNames.reduce((scaleObject:any, columnName) => {
      if (this.scalers.has(columnName)){
        scaleObject[ columnName ] = this.scalers.get(columnName).descale(data[ columnName ]);
      }
      return scaleObject;
    }, {});
    const labeledData = columnNames.reduce((labelObject:any, columnName) => {
      if (this.labels.has(columnName)){
        labelObject[ columnName ] = this.labels.get(columnName).get(data[ columnName ]);
      }
      return labelObject;
    }, {});
    const encodedData = columnNames.reduce((encodedObject, columnName) => {
      if (this.encoders.has(columnName)) {
        const encoded = this.oneHotDecoder(columnName, {
          data: [data,],
        });
        // console.log({encoded})
        encodedObject = Object.assign({}, encodedObject, encoded[ 0 ]);
        if (config.removeValues) {
          removedColumns.push(...this.encoders.get(columnName).labels.map((label: any)=>`${this.encoders.get(columnName).prefix}${label}`));
        }
      }
      return encodedObject;
    }, {});
    transformedObject = Object.assign(transformedObject, scaledData, labeledData, encodedData);
    if (config.removeValues && removedColumns.length) {
      transformedObject = Object.keys(transformedObject).reduce((removedObject:any, propertyName) => {
        if (removedColumns.indexOf(propertyName) === -1) {
          removedObject[ propertyName ] = transformedObject[ propertyName ];
        }
        return removedObject;
      }, {});
    }
    return transformedObject;
  }
  /**
   * transforms an object and replaces values that have been scaled or encoded
   * @example
DataSet.transformObject({
  'Country': 'Brazil',
  'Age': '44',
  'Salary': '72000',
  'Purchased': 'N',
}); // =>
// { 
//  Country: 'Brazil',
//  Age: 3.784189633918261,
//  Salary: '72000',
//  Purchased: 'N',
//  Country_Brazil: 1,
//  Country_Mexico: 0,
//  Country_Ghana: 0
// }
   * @param data 
   * @param options 
   * @returns {Object} 
   */
  transformObject(data: { [x: string]: any; }, options: {}) {
    const config = Object.assign({
      removeValues: false,
      checkColumnLength: true,
    }, options);
    const removedColumns: string[] = [];
    // if (Array.isArray(data)) return data.map(datum => this.transformObject);
    const encodedColumns = new Array().concat(...Array.from(this.encoders.keys())
      .map(encodedColumn => this.encoders.get(encodedColumn).labels
        .map((label: any)=>`${this.encoders.get(encodedColumn).prefix}${label}`)
      )
    );
    const currentColumns = (this.data.length)
      ? Object.keys(this.data[ 0 ])
      : Object.keys(data);
    const objectColumns = Object.keys(data).concat(encodedColumns);
    // console.log({ encodedColumns,currentColumns,objectColumns });
    const differentKeys = objectColumns.reduce((diffKeys:any[], val:any) => {
      if (currentColumns.indexOf(val) === -1 && encodedColumns.indexOf(val) === -1) diffKeys.push(val);
      return diffKeys;
    }, []);
    let transformedObject = Object.assign({}, data);
    if (config.checkColumnLength && currentColumns.length !== objectColumns.length && currentColumns.length+encodedColumns.length !== objectColumns.length ) {
      throw new RangeError(`Object must have the same number of keys (${objectColumns.length}) as data in your dataset(${currentColumns.length})`);
    } else if (config.checkColumnLength && differentKeys.length) {
      throw new ReferenceError(`Object must have identical keys as data in your DataSet. Invalid keys: ${differentKeys.join(',')}`);
    } else {
      const scaledData = objectColumns.reduce((scaleObject:any, columnName) => {
        if (this.scalers.has(columnName)){
          scaleObject[ columnName ] = this.scalers.get(columnName).scale(data[ columnName ]);
        }
        return scaleObject;
      }, {});
      const labeledData = objectColumns.reduce((labelObject:any, columnName) => {
        if (this.labels.has(columnName)){
          labelObject[ columnName ] = this.labels.get(columnName).get(data[ columnName ]);
        }
        return labelObject;
      }, {});
      const encodedData = objectColumns.reduce((encodedObject, columnName) => {
        if (this.encoders.has(columnName)) {
          encodedObject = Object.assign({}, encodedObject, this.encodeObject(data, this.encoders.get(columnName)));
          if (config.removeValues) {
            removedColumns.push(columnName);
          }
        }
        return encodedObject;
      }, {});
      transformedObject = Object.assign(transformedObject, scaledData, labeledData, encodedData);
      if (config.removeValues && removedColumns.length) {
        transformedObject = Object.keys(transformedObject).reduce((removedObject:any, propertyName) => {
          if (removedColumns.indexOf(propertyName) === -1) removedObject[ propertyName ] = transformedObject[ propertyName ];
          return removedObject;
        }, {});
      }
    }
    return transformedObject;
  } 
  /**
   * returns a new array of a selected column from an array of objects and replaces empty values, encodes values and scales values
   * @example
   * //column Replace returns new Array with replaced missing data
//[ '44','27','30','38','40','35',38.77777777777778,'48','50','37' ]
const ReplacedAgeMeanColumn = dataset.columnReplace('Age',{strategy:'mean'});
  * @param {string} name - csv column header, or JSON object property name 
  * @param options 
  * @param {boolean} [options.empty=true] - replace empty values 
  * @param {boolean} [options.strategy="mean"] - strategy for replacing value, any array stat method from ml.js (mean, standardDeviation, median) or (label,labelEncoder,onehot,oneHotEncoder)
  * @returns {array|Object[]}
  */
  columnReplace(name: any, options:any = {}) {
    const config = Object.assign({
      strategy: 'mean',
      empty: true,
      arrayOptions: {
        parseFloat: true,
        filter: (val: any) => val,
      },
      labelOptions: {},
    }, options);
    let replaceVal: any;// { [x: string]: any;[x: number]: any; } | undefined;
    let replace = {
      test: (val: any) => !val,
      value: replaceVal,
    };
    switch (config.strategy) {
    case 'scale':
      replaceVal = this.columnScale(name, config.scaleOptions);
      replace = {
        test: val => true,
        value: (result: any, val: any, index: string | number, arr: any) => replaceVal[index],
      };
      break;
    case 'descale':
      replaceVal = this.columnDescale(name, config.descaleOptions);
      replace = {
        test: val => true,
        value: (result: any, val: any, index: string | number, arr: any) => replaceVal[index],
      };
      break;
    case 'label':
    case 'labelEncoder':
      replaceVal = this.labelEncoder(name, config.labelOptions);
      replace = {
        test: val => true,
        value: (result: any, val: any, index: string | number, arr: any) => replaceVal[index],
      };
      break;
    case 'labeldecode':
    case 'labelDecode':
    case 'labelDecoder':
      replaceVal = this.labelDecode(name, config.labelOptions);
      replace = {
        test: val => true,
        value: (result: any, val: any, index: string | number, arr: any) => replaceVal[index],
      };
      break;
    case 'onehot':
    case 'oneHot':
    case 'oneHotEncode':
    case 'oneHotEncoder':
      replaceVal = this.oneHotEncoder(name, config.oneHotOptions);
      replace = {
        test: val => true,
        value: (result: any, val: any, index: string | number, arr: any) => replaceVal[index],
      };
      return replaceVal;
      // break;
    case 'reducer':
    case 'reduce':
      replaceVal = this.columnReducer(name, config.reducerOptions); 
      return replaceVal;  
    case 'merge':
      replaceVal = this.columnMerge(name, config.mergeData); 
      return replaceVal; 
    case 'parseNumber':
      replaceVal = this.columnArray(name).map((num: any) => Number(num)); 
      return replaceVal; 
    default:
      replaceVal = ArrayStat[config.strategy](this.columnArray(name, config.arrayOptions));
      replace.value = replaceVal;
      break;
    }
    return this.columnArray(name,
      Object.assign({}, {
        replace,
        scale: options.scale,
      }, options.columnOptions));
  }
  /**
     * mutates data property of DataSet by replacing multiple columns in a single command
     * @example
     * //fit Columns, mutates dataset
dataset.fitColumns({
  columns:[{name:'Age',options:{ strategy:'mean'} }]
});
// dataset
// class DataSet
//   data:[
//     {
//       'Country': 'Brazil',
//       'Age': '38.77777777777778',
//       'Salary': '72000',
//       'Purchased': 'N',
//     }
//     ...
//   ]
  * @param {Boolean} options.returnData - return updated DataSet data property 
  * @param {Object[]} options.columns - {name:'columnName',options:{strategy:'mean',labelOoptions:{}},}
  * @returns {Object[]}
  */
  fitColumns(options:any = {}, mockDataOptions = {}) {
    const config = Object.assign({
      returnData:true,
      columns: [],
    }, options);
    if ( !options.columns || Array.isArray(options.columns) ===false) {
      config.columns = (options.columns)
        ? DataSet.getTransforms(options.columns)
        : DataSet.getTransforms(options);
    }
    const fittedColumns = config.columns
      .reduce((result:any, val:any, index:number, arr:any[]) => {
        let replacedColumn = this.columnReplace(val.name, val.options);
        if (Array.isArray(replacedColumn)) {
          replacedColumn = replacedColumn.map(columnVal => ({
            [val.name]: columnVal,
          }));
          result[val.name] = replacedColumn;
        } else {
          Object.keys(replacedColumn).forEach(repColName => {
            result[repColName] = replacedColumn[repColName].map((columnVal: any) => ({
              [repColName]: columnVal,
            }));
          });
        }
        return result;
      }, {});
    if (Object.keys(fittedColumns) && Object.keys(fittedColumns).length) {
      const columnNames = Object.keys(fittedColumns);
      const fittedData = fittedColumns[columnNames[0]]
        .reduce((result: {}[], val: any, index: string | number, arr: any) => {
          const returnObj:any = {};
          columnNames.forEach(colName => {
            returnObj[colName] = fittedColumns[colName][index][colName];
          });
          result.push(returnObj);
          return result;
        }, []);
      this.data = this.data.map((val, index) => Object.assign({}, val, fittedData[index]));
    }
    return config.returnData ? this.data : this;
  }
  /**
   * Mutate dataset data by inversing all transforms
   * @example
DataSet.data;
// [{ 
//  Country: 'Brazil',
//  Age: 3.784189633918261,
//  Salary: '72000',
//  Purchased: 'N',
//  Country_Brazil: 1,
//  Country_Mexico: 0,
//  Country_Ghana: 0
// },
// ...
// ]
DataSet.fitInverseTransforms(); // =>
// [{
//   'Country': 'Brazil',
//   'Age': '44',
//   'Salary': '72000',
//   'Purchased': 'N',
// },
// ...
// ]
   * @param options 
   */
  fitInverseTransforms(options:any = {}) {
    const config = Object.assign({
      returnData: true,
    }, options);
    this.data = this.data.map(val => {
      return (options.removeValues)
        ? this.inverseTransformObject(val, options)
        : Object.assign({}, val, this.inverseTransformObject(val, options));
    });
    return config.returnData ? this.data : this;
  }
  /**
   * Mutate dataset data with all transforms
   * @param options
   * @example
DataSet.data;
// [{
//   'Country': 'Brazil',
//   'Age': '44',
//   'Salary': '72000',
//   'Purchased': 'N',
// },
// ...
// ]
DataSet.fitTransforms(); // =>
// [{ 
//  Country: 'Brazil',
//  Age: 3.784189633918261,
//  Salary: '72000',
//  Purchased: 'N',
//  Country_Brazil: 1,
//  Country_Mexico: 0,
//  Country_Ghana: 0
// },
// ...
// ] 
   */
  fitTransforms(options:any = {}) {
    const config = Object.assign({
      returnData: true,
    }, options);
    this.data = this.data.map(val => {
      return (options.removeValues)
        ? this.transformObject(val, options)
        : Object.assign({}, val, this.transformObject(val, options));
    });
    return config.returnData ? this.data : this;
  }
}