# All files / utils rawFluxDataTable.ts

83.64% Statements 46/55
57.14% Branches 8/14
75% Functions 6/8
81.25% Lines 39/48

Press n or j to go to the next uncovered block, b, p or k for the previous block.

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    81
    82
    83
    84
    85
    86
    87
    88
    89
    90
    91
    92
    93
    94
    95
    96
    97
    98
    99
    100
    101
    102
    103
    104
    105
    106
    107
    108
    109
    110
    111
    112
    113
    114
    115
    116
    117
    118
    119
    120
    121
    122
    123
    124
    125
    126
    127
    128
    129
    130
    131
    132
    1331x
    1x
    1x

    1x
    1x

    1x
    2x
    4x
    4x
    2x

    2x
    4x

    2x

    4x

    28x

    2x

    1x

    3x
    3x
    6x
    6x
    6x
    6x

    6x
    6x
    6x
    6x

    6x

    6x
    6x

    6x

    6x
    3x

    3x
    6x

    3x

    6x

    31x

    3x

     import Papa from 'papaparse'
    import HtmlEntities from 'he'
    import unraw from 'unraw'

    import {parseChunks} from './fluxParsing'
    import {
      CSV_OBJECT_BASE_NAME,
      CSV_OBJECT_START_STRING,
      CSV_OBJECT_END_STRING,
    } from '../constants/rawFluxData'

    export interface ParseFilesResult {
      data: string[][]
      maxColumnCount: number
    }

    export const parseFiles = (responses: string[]): ParseFilesResult => {
      const chunks = parseChunks(responses.join('\n\n'))
      const parsedChunks = chunks.map(c => Papa.parse(c).data)
      const maxColumnCount = Math.max(...parsedChunks.map(c => c[0].length))
      const data = []

      for (let i = 0; i < parsedChunks.length; i++) {
        if (i !== 0) {
          // Seperate each chunk by an empty line, just like in the unparsed CSV
          data.push([])
        }

        for (let j = 0; j < parsedChunks[i].length; j++) {
          // Danger zone! Since the contents of each chunk are potentially quite
          // large, the contents need to be concated using a loop rather than with
          // `concat`, a splat or similar. Otherwise we see a "Maximum call size
          // exceeded" error for large CSVs
          data.push(parsedChunks[i][j])
        }
      }

      return {data, maxColumnCount}
    }

    export const parseFilesWithObjects = (
      responses: string[]
    ): ParseFilesResult => {
      const chunks = parseChunks(responses.join('\n\n'))
      const parsedChunks = chunks.map(c => {
        const objList = []
        let updatedString = c
        let startIndex = 0
        let endIndex = 0

        /* ********************************************************************
         * Parse objects in the CSV by decoding the html and removing
         * the following inside the objects, so that the stringified version is
         * parsable by JSON.parse and becomes useful to the user:
         *   - double double-quotes
         *   - escaped double-quotes
         *   - newlines
         *   - tabs
         *   - escaped null
         */
        while (startIndex !== -1 && endIndex !== -1) {
          startIndex = updatedString.indexOf(CSV_OBJECT_START_STRING)
          endIndex = updatedString.indexOf(CSV_OBJECT_END_STRING)
          Iif (startIndex !== -1 && endIndex !== -1) {
            objList.push(
              unraw(
                HtmlEntities.decode(
                  updatedString.slice(
                    startIndex + CSV_OBJECT_START_STRING.length - 1,
                    endIndex + 1
                  )
                )
                  .replace(/""/g, '"')
                  .replace(/:",/g, ':"",')
                  .replace(/"","}/g, '","}')
                  .replace(/\\"/g, '')
                  .replace(/\\n/g, ' ')
                  .replace(/\\t/g, '')
                  .replace(/\\u0000/g, '')
              )
            )
            updatedString =
              updatedString.slice(0, startIndex + 1) +
              `${CSV_OBJECT_BASE_NAME}${objList.length - 1}` +
              updatedString.slice(endIndex + CSV_OBJECT_END_STRING.length - 1)
          }
        }

        const {data} = Papa.parse(updatedString)

        // After we parse the CSV, if we have a list of parsed objects
        //   go back and re-insert them
        let iterator = 0
        objList.forEach((obj, objIndex) => {
          let dataIndex = -1
          while (iterator < data.length && dataIndex === -1) {
            dataIndex = data[iterator].findIndex(
              element => element === `${CSV_OBJECT_BASE_NAME}${objIndex}`
            )
            if (dataIndex > -1) {
              data[iterator][dataIndex] = data[iterator][dataIndex].replace(
                `${CSV_OBJECT_BASE_NAME}${objIndex}`,
                obj
              )
            } else {
              iterator += 1
            }
          }
        })

        return data
      })
      const maxColumnCount = Math.max(...parsedChunks.map(c => c[0].length))
      const data = []

      for (let i = 0; i < parsedChunks.length; i++) {
        if (i !== 0) {
          // Separate each chunk by an empty line, just like in the unparsed CSV
          data.push([])
        }

        for (let j = 0; j < parsedChunks[i].length; j++) {
          // Danger zone! Since the contents of each chunk are potentially quite
          // large, the contents need to be concated using a loop rather than with
          // `concat`, a splat or similar. Otherwise we see a "Maximum call size
          // exceeded" error for large CSVs
          data.push(parsedChunks[i][j])
        }
      }

      return {data, maxColumnCount}
    }