# All files / src/resource-selection-optimizer in-clause-under-conjunction.ts

94.07% Statements 111/118
79.41% Branches 27/34
100% Functions 20/20
95.58% Lines 108/113

Press n or j to go to the next uncovered block, b, p or k for the previous block.

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    81
    82
    83
    84
    85
    86
    87
    88
    89
    90
    91
    92
    93
    94
    95
    96
    97
    98
    99
    100
    101
    102
    103
    104
    105
    106
    107
    108
    109
    110
    111
    112
    113
    114
    115
    116
    117
    118
    119
    120
    121
    122
    123
    124
    125
    126
    127
    128
    129
    130
    131
    132
    133
    134
    135
    136
    137
    138
    139
    140
    141
    142
    143
    144
    145
    146
    147
    148
    149
    150
    151
    152
    153
    154
    155
    156
    157
    158
    159
    160
    161
    162
    163
    164
    165
    166
    167
    168
    169
    170
    171
    172
    173
    174
    175
    176
    177
    178
    179
    180
    181
    182
    183
    184
    185
    186
    187
    188
    189
    190
    191
    192
    193
    194
    195
    196
    197
    198
    199
    200
    201
    202
    203
    204
    205
    206
    207
    208
    209
    210
    211
    212
    213
    214
    215
    216
    217
    218
    219
    220
    2211x

    1x
    1x
    1x
    1x
    1x
    1x
    1x

    1x
    1x
    1x
    1x
    1x
    1x

    1x
    7x

    7x
    7x
    7x
    7x
    7x
    7x

    10x

    10x

    3x

    3x
    3x

    3x
    3x
    3x
    3x
    3x
    3x
    3x
    3x

    3x

    3x

    3x
    7x
    3x

    3x
    6x
    6x

    6x

    6x

    204x

    3x
    3x
    3x

    3x
    243x
    603x

    603x
    94x
    150x

    150x
    150x
    150x

    3x

    3x
    3x
    20x
    20x

    3x

    805x
    18796x
    592x

    3x
    3x

    3x
    20x
    805x
    805x

    805x
    213x

    805x

    805x
    805x

    3x

    3x

    3x
    6x

    6x

    204x

    204x
    204x
    5483x
    171346x
    166034x
    166034x
    166034x
    14x
    2x

    166020x

    6x

    3x

    3x

    3x

    3x
    3x
    22x
    22x

    3x
    3x

    3x
    3x

    3x

    7x

     import * as head from 'lodash.head';
    import * as values from 'lodash.values';
    import * as keys from 'lodash.keys';
    import * as get from 'lodash.get';
    import * as isEmpty from 'lodash.isempty';
    import * as startsWith from 'lodash.startswith';
    import * as includes from 'lodash.includes';
    import * as compact from 'lodash.compact';
    import { DdfCsvError } from '../ddfcsv-error';
    import { IDatapackage, IResourceSelectionOptimizer, IResourceRead, IBaseReaderOptions } from '../interfaces';
    import { QueryFeature, featureDetectors, IQuery } from 'ddf-query-validator';

    const WHERE_KEYWORD = 'where';
    const JOIN_KEYWORD = 'join';
    const KEY_IN = '$in';
    const KEY_NIN = '$nin';
    const KEY_AND = '$and';
    const KEY_OR = '$or';

    export class InClauseUnderConjunction implements IResourceSelectionOptimizer {
      private flow: any = {};
      private fileReader: IResourceRead;
      private datasetPath: string;
      private query: IQuery;
      private datapackage: IDatapackage;
      private conceptsLookup;

      constructor(private parent, queryParam, private options: IBaseReaderOptions) {
        this.fileReader = options.fileReader;
        this.datasetPath = options.basePath;
        this.query = queryParam;
        this.datapackage = options.datapackage;
        this.conceptsLookup = options.conceptsLookup;
      }

      isMatched(): boolean {
        this.flow.joinObject = get(this.query, JOIN_KEYWORD);

        return this.query.from === "datapoints" && this.flow.joinObject;
      }

      async getRecommendedFilesSet(): Promise<string[]> {
        const { debug, error, warning } = this.options.diagnostic.prepareDiagnosticFor('getRecommendedFilesSet');

        Eif (this.isMatched()) {
          debug('plugin matched');

          let result;
          try {
            this.flow.processableClauses = await this.collectProcessableClauses();
            Iif (!this.flow.processableClauses) return [];
            this.collectEntityFilesNames();
            const data = await this.collectEntities();
            this.fillEntityValuesHash(data);
            this.getFilesGroupsQueryClause();
            result = this.getOptimalFilesGroup();
          } catch (err) {
            error('wrong data processing', err);
            return [];
          }

          debug('recommended files found', result);

          return result;
        } else {
          const message = `Plugin "InClauseUnderConjunction" is not matched!`;
          warning(message);
          throw new DdfCsvError(message, 'InClauseUnderConjunction plugin');
        }
      }

      private collectProcessableClauses(): Promise<any> {
        const joinKeys = keys(this.flow.joinObject).filter(
          key => ["entity_domain", "entity_set"].includes(this.options.conceptsLookup.get(key.slice(1))?.concept_type));
        Iif (!joinKeys.length) return Promise.resolve(false);

        return Promise.all(joinKeys.map(joinKey => {
          const key = this.flow.joinObject[joinKey].key;
          const where = get(this.flow.joinObject, `${joinKey}.${WHERE_KEYWORD}`, {});

          return this.parent.queryData({
            select: { key: [key] },
            where,
            from: this.options.conceptsLookup.has(key) ? 'entities' : 'concepts'
          }, Object.assign({ joinID: joinKey }, this.options))
            .then(result => ({
              key,
              entities: new Set(result.map(row => row[ key ]))
            }));
        }));

      }

      private collectEntityFilesNames(): InClauseUnderConjunction {
        this.flow.entityFilesNames = new Set();
        this.flow.entityResources = new Set();
        this.flow.fileNameToPrimaryKeyHash = new Map();

        for (const schemaResourceRecord of this.datapackage.ddfSchema.entities) {
          for (const clause of this.flow.processableClauses) {
            const key = clause.key;

            if (head(schemaResourceRecord.primaryKey) === key) {
              for (const resourceName of schemaResourceRecord.resources) {
                const resource = this.options.resourcesLookup.get(resourceName);

                this.flow.entityResources.add(resource);
                this.flow.entityFilesNames.add(resource.path);
                this.flow.fileNameToPrimaryKeyHash.set(resource.path, key);
              }
            }
          }
        }

        return this;
      }

      private collectEntities(): Promise<any> {
        const self = this;
        const actions = [...self.flow.entityResources].map(resource => {
          return (resource.data || (resource.data = self.parent.loadFile(resource.path, self.options)))
            .then(data => ({result: data, file: resource.path}));
        });

        return Promise.all(actions);
      }

      private fillEntityValuesHash(entitiesData): InClauseUnderConjunction {
        const getSubdomainsFromRecord = record => compact(keys(record)
          .filter(key => startsWith(key, 'is--') && (record[key] === true))
          .map(key => key.replace(/^is--/, '')));

        this.flow.entityValueToFileHash = new Map();
        this.flow.entityValueToDomainHash = new Map();

        for (const entityFileDescriptor of entitiesData) {
          for (const entityRecord of entityFileDescriptor.result.data) {
            const primaryKeyForThisFile = this.flow.fileNameToPrimaryKeyHash.get(entityFileDescriptor.file);
            const domainsForCurrentRecord = [...getSubdomainsFromRecord(entityRecord)];

            if (isEmpty(domainsForCurrentRecord)) {
              domainsForCurrentRecord.push(primaryKeyForThisFile);
            }

            const primaryKeyCellValue = entityRecord[primaryKeyForThisFile] || entityRecord[domainsForCurrentRecord[0]];

            this.flow.entityValueToDomainHash.set(primaryKeyCellValue, domainsForCurrentRecord);
            this.flow.entityValueToFileHash.set(primaryKeyCellValue, entityFileDescriptor.file);
          }
        }

        return this;
      }

      private getFilesGroupsQueryClause(): InClauseUnderConjunction {
        const filesGroupsByClause = new Map();

        for (const clause of this.flow.processableClauses) {
          const filesGroupByClause = {
            datapoints: new Set()
          };

          for (const entityValueFromClause of clause.entities) {

            const entitiesByQuery = this.flow.entityValueToDomainHash.get(entityValueFromClause);

            for (const entityByQuery of entitiesByQuery) {
              for (const schemaResourceRecord of this.datapackage.ddfSchema.datapoints) {
                for (const resourceName of schemaResourceRecord.resources) {
                  if (includes(schemaResourceRecord.primaryKey, entityByQuery)) {
                    const resource = this.options.resourcesLookup.get(resourceName);
                    const constraint = resource.constraints?.[entityByQuery];
                    if ( constraint ) {
                      if (constraint.includes(entityValueFromClause)) {
                        filesGroupByClause.datapoints.add(resource.path);
                      }
                    } else {
                      filesGroupByClause.datapoints.add(resource.path);
                    }
                  }
                }
              }
            }
          }

          filesGroupsByClause.set(clause.key, filesGroupByClause);
        }

        this.flow.filesGroupsByClause = filesGroupsByClause;

        return this;
      }

      private getOptimalFilesGroup(): string[] {

        const entities = this.flow.entityFilesNames;

        const concepts = new Set();
        for (const schemaResourceRecord of this.datapackage.ddfSchema.concepts) {
          for (const resourceName of schemaResourceRecord.resources) {
            concepts.add(this.options.resourcesLookup.get(resourceName).path);
          }
        }

        const clauseKeys = this.flow.filesGroupsByClause.keys();
        let datapoints = Array.from(this.flow.filesGroupsByClause.get(clauseKeys.next().value).datapoints);

        for (const key of clauseKeys) {
          datapoints = this.intersectArray(datapoints, Array.from(this.flow.filesGroupsByClause.get(key).datapoints));
        }

        return [...Array.from(concepts), ...Array.from(entities)].concat(datapoints) as string[];
      }

      private intersectArray(array1, array2) {
        return array1.filter(value => array2.includes(value));
      }

    }