const fs = require('fs-extra');
const path = require('path');
const readXlsxFile = require('read-excel-file/node');

async function getFiles(dirpath = './data') {
  const allFiles = await fs.readdir(dirpath);
  const xslxFiles = allFiles.filter(x => x.indexOf('.xlsx') !== -1 && x.indexOf('~') !== 0);
  return xslxFiles.map(file => path.join(dirpath, file));
}

async function convertFile(filepath) {
  const toFilterValue = x => x.toLowerCase().replace(/\s+/g, '');
  const outputFilename = filepath.substring(0, filepath.lastIndexOf('.') + 1) + 'json';
  console.log(`Reading file "${filepath}" and converting to "${outputFilename}"`);
  const sheets = await readXlsxFile(filepath, { getSheets: true });
  let dataRows = [];
  let sheetsData = await Promise.all(
    sheets.map(async sheet => {
      return await readXlsxFile(filepath, { sheet: sheet.name });
    }),
  ); //await readXlsxFile(filepath, { sheet: sheet.name });
  const filterColumnNames = ['what', 'deliveryscope'];
  let types = {};
  sheetsData.forEach(sheetData => {
    const firstRow = sheetData.shift().map(x => toFilterValue(x));
    // Set up column for adding options
    firstRow.forEach(async (col, i) => {
      if (typeof types[col] === 'undefined' && filterColumnNames.includes(col)) {
        types[col] = [];
      }
    });
    sheetData.forEach((row, rowindex) => {
      const dataRow = {};
      firstRow.forEach(async (col, i) => {
        const rowVal = row[i];
        dataRow[col] = rowVal;
        // Add to types options
        if (filterColumnNames.includes(col)) {
          const filterVal = toFilterValue(rowVal);
          dataRow[col] = filterVal;
          if (!types[col].includes(filterVal)) {
            types[col].push(filterVal);
          }
        }
      });
      dataRows.push(dataRow);
    });
  });
  const dataFile = filepath;
  const fileData = {
    file: filepath,
    types: types,
    data: dataRows,
  };
  console.log(`Stored ${dataRows.length} rows from ${sheetsData.length} sheets. ${Object.keys(types).length} filter types identified`);
  fs.writeJsonSync(outputFilename, fileData);
}

async function convert(dirpath = './data') {
  const files = await getFiles(dirpath);
  files.forEach(convertFile);
}
