"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.getSchemaStructureFromDbtModels = exports.attachTypesToModels = exports.convertExplores = exports.convertTable = void 0;
const tslib_1 = require("tslib");
const merge_1 = tslib_1.__importDefault(require("lodash/merge"));
const partition_1 = tslib_1.__importDefault(require("lodash/partition"));
const dbt_1 = require("../types/dbt");
const errors_1 = require("../types/errors");
const explore_1 = require("../types/explore");
const field_1 = require("../types/field");
const filterGrammar_1 = require("../types/filterGrammar");
const table_1 = require("../types/table");
const assertUnreachable_1 = tslib_1.__importDefault(require("../utils/assertUnreachable"));
const timeFrames_1 = require("../utils/timeFrames");
const exploreCompiler_1 = require("./exploreCompiler");
const lightdashProjectConfig_1 = require("./lightdashProjectConfig");
const convertTimezone = (timestampSql, default_source_tz, target_tz, adapterType) => {
    // todo: implement default_source_tz
    // todo: implement target_tz
    // todo: implement conversion for all adapters
    switch (adapterType) {
        case dbt_1.SupportedDbtAdapter.BIGQUERY:
            // TIMESTAMPS: stored as utc. returns utc. convert from utc to target_tz
            //   DATETIME: no tz. assume default_source_tz. covert from default_source_tz to target_tz
            return timestampSql;
        case dbt_1.SupportedDbtAdapter.SNOWFLAKE:
            // TIMESTAMP_NTZ: no tz. assume default_source_tz. convert from default_source_tz to target_tz
            // TIMESTAMP_LTZ: stored in utc. returns in session tz. convert from session tz to target_tz
            // TIMESTAMP_TZ: stored with tz. returns with tz. convert from value tz to target_tz
            return `TO_TIMESTAMP_NTZ(CONVERT_TIMEZONE('UTC', ${timestampSql}))`;
        case dbt_1.SupportedDbtAdapter.REDSHIFT:
            // TIMESTAMP WITH TIME ZONE: stored in utc. returns utc. convert from utc to target_tz
            // TIMESTAMP WITHOUT TIME ZONE: no tz. assume utc. convert from utc to target_tz
            return timestampSql;
        case dbt_1.SupportedDbtAdapter.POSTGRES:
            // TIMESTAMP WITH TIME ZONE: stored as utc. returns in session tz. convert from session tz to target tz
            // TIMESTAMP WITHOUT TIME ZONE: no tz. assume default_source_tz. convert from default_source_tz to target_tz
            return timestampSql;
        case dbt_1.SupportedDbtAdapter.DUCKDB:
            return timestampSql;
        case dbt_1.SupportedDbtAdapter.DATABRICKS:
        case dbt_1.SupportedDbtAdapter.SPARK:
            return timestampSql;
        // Athena uses Trino SQL, timestamps return in server timezone
        case dbt_1.SupportedDbtAdapter.TRINO:
        case dbt_1.SupportedDbtAdapter.ATHENA:
            return timestampSql;
        case dbt_1.SupportedDbtAdapter.CLICKHOUSE:
            // DateTime: stored in server timezone, returns in server timezone
            return timestampSql;
        default:
            return (0, assertUnreachable_1.default)(adapterType, new errors_1.ParseError(`Cannot recognise warehouse ${adapterType}`));
    }
};
const isInterval = (dimensionType, dimension) => [field_1.DimensionType.DATE, field_1.DimensionType.TIMESTAMP].includes(dimensionType) &&
    dimension?.time_intervals !== false &&
    ((dimension?.time_intervals && dimension.time_intervals !== 'OFF') ||
        !dimension?.time_intervals);
const convertDimension = (index, targetWarehouse, model, tableLabel, column, source, timeInterval, startOfWeek, isAdditionalDimension, disableTimestampConversion) => {
    // Config block takes priority, then meta block
    const meta = (0, merge_1.default)({}, column.meta, column.config?.meta);
    let type = meta.dimension?.type || column.data_type || field_1.DimensionType.STRING;
    if (!Object.values(field_1.DimensionType).includes(type)) {
        throw new errors_1.MissingCatalogEntryError(`Could not recognise type "${type}" for dimension "${column.name}" in dbt model "${model.name}". Valid types are: ${Object.values(field_1.DimensionType).join(', ')}`, {});
    }
    let name = meta.dimension?.name || column.name;
    let sql = meta.dimension?.sql || (0, field_1.defaultSql)(column.name);
    let label = meta.dimension?.label || (0, field_1.friendlyName)(name);
    if (type === field_1.DimensionType.TIMESTAMP && !disableTimestampConversion) {
        sql = convertTimezone(sql, 'UTC', 'UTC', targetWarehouse);
    }
    const isIntervalBase = timeInterval === undefined && isInterval(type, meta.dimension);
    let timeIntervalBaseDimensionName;
    let timeIntervalBaseDimensionType;
    const groups = (0, dbt_1.convertToGroups)(meta.dimension?.groups, meta.dimension?.group_label);
    if (timeInterval) {
        timeIntervalBaseDimensionName = name;
        timeIntervalBaseDimensionType = type;
        sql = timeFrames_1.timeFrameConfigs[timeInterval].getSql(targetWarehouse, timeInterval, sql, type, startOfWeek);
        name = `${column.name}_${timeInterval.toLowerCase()}`;
        label = `${label} ${timeFrames_1.timeFrameConfigs[timeInterval]
            .getLabel()
            .toLowerCase()}`;
        groups.push(meta.dimension?.label ??
            (0, field_1.friendlyName)(timeIntervalBaseDimensionName));
        type = timeFrames_1.timeFrameConfigs[timeInterval].getDimensionType(type);
    }
    return {
        index,
        fieldType: field_1.FieldType.DIMENSION,
        name,
        label,
        sql,
        table: model.name,
        tableLabel,
        type,
        description: meta.dimension?.description || column.description,
        source,
        timeInterval,
        timeIntervalBaseDimensionName,
        timeIntervalBaseDimensionType,
        hidden: !!meta.dimension?.hidden,
        format: meta.dimension?.format,
        round: meta.dimension?.round,
        compact: meta.dimension?.compact,
        separator: meta.dimension?.separator,
        requiredAttributes: meta.dimension?.required_attributes,
        anyAttributes: meta.dimension?.any_attributes,
        colors: meta.dimension?.colors,
        ...(meta.dimension?.urls ? { urls: meta.dimension.urls } : {}),
        ...(meta.dimension?.image ? { image: meta.dimension.image } : {}),
        ...(meta.dimension?.richText
            ? { richText: meta.dimension.richText }
            : {}),
        ...(isAdditionalDimension ? { isAdditionalDimension } : {}),
        // Polarity flip: YAML reads `convert_timezone: false` (defaults true,
        // matches dbt convention); in-memory we store the inverse so truthiness
        // matches the semantic — `if (dim.skipTimezoneConversion)` is correct,
        // no `=== false` trap, and absent collapses to the default.
        ...(meta.dimension?.convert_timezone === false
            ? { skipTimezoneConversion: true }
            : {}),
        groups,
        isIntervalBase,
        ...(meta.dimension && meta.dimension.tags
            ? {
                tags: Array.isArray(meta.dimension.tags)
                    ? meta.dimension.tags
                    : [meta.dimension.tags],
            }
            : {}),
        ...(meta.dimension?.ai_hint
            ? { aiHint: (0, dbt_1.convertToAiHints)(meta.dimension.ai_hint) }
            : {}),
        ...(meta.dimension?.case_sensitive !== undefined
            ? { caseSensitive: meta.dimension.case_sensitive }
            : {}),
        ...(meta.dimension?.spotlight?.filter_by === false ||
            meta.dimension?.spotlight?.segment_by === false
            ? {
                spotlight: {
                    ...(meta.dimension?.spotlight?.filter_by === false && {
                        filterBy: false,
                    }),
                    ...(meta.dimension?.spotlight?.segment_by === false && {
                        segmentBy: false,
                    }),
                },
            }
            : {}),
    };
};
/**
 * Convert an explore-scoped additional dimension to Dimension(s).
 * Reuses convertDimension by creating a synthetic column object.
 * Returns the base dimension plus any time interval dimensions.
 */
const convertExploreScopedDimension = (index, tableName, tableLabel, dimensionName, dimensionConfig, targetWarehouse, startOfWeek) => {
    // Create a synthetic column to reuse convertDimension
    const syntheticColumn = {
        name: dimensionName,
        data_type: dimensionConfig.type,
        meta: {
            dimension: dimensionConfig,
        },
    };
    const syntheticModel = {
        name: tableName,
        relation_name: tableName,
    };
    const baseDimension = convertDimension(index, targetWarehouse, syntheticModel, tableLabel, syntheticColumn, undefined, undefined, startOfWeek, true);
    const result = {
        [dimensionName]: baseDimension,
    };
    // Process time intervals if applicable (same logic as column-level additional dimensions)
    if (baseDimension.isIntervalBase) {
        let intervals = [];
        if (dimensionConfig.time_intervals &&
            Array.isArray(dimensionConfig.time_intervals)) {
            intervals = (0, timeFrames_1.validateTimeFrames)(dimensionConfig.time_intervals);
        }
        else {
            intervals = (0, timeFrames_1.getDefaultTimeFrames)(dimensionConfig.type);
        }
        intervals.forEach((interval) => {
            const intervalDimension = convertDimension(index, targetWarehouse, syntheticModel, tableLabel, syntheticColumn, undefined, interval, startOfWeek, true);
            result[intervalDimension.name] = intervalDimension;
        });
    }
    return result;
};
const generateTableLineage = (model, depGraph) => {
    const modelFamilyIds = [
        ...depGraph.dependantsOf(model.unique_id),
        ...depGraph.dependenciesOf(model.unique_id),
        model.unique_id,
    ];
    return modelFamilyIds.reduce((prev, nodeId) => ({
        ...prev,
        [depGraph.getNodeData(nodeId).name]: depGraph
            .directDependenciesOf(nodeId)
            .map((d) => depGraph.getNodeData(d)),
    }), {});
};
/**
 * @deprecated This function uses the old dbt metrics format.
 */
const convertDbtMetricToLightdashMetric = (metric, tableName, tableLabel, spotlightConfig, modelCategories) => {
    let sql;
    let type;
    if (metric.calculation_method === 'derived') {
        type = field_1.MetricType.NUMBER;
        const referencedMetrics = new Set((metric.metrics || []).map((m) => m[0]));
        if (!metric.expression) {
            throw new errors_1.ParseError(`dbt derived metric "${metric.name}" must have the expression field set`);
        }
        sql = metric.expression;
        referencedMetrics.forEach((ref) => {
            const re = new RegExp(`\\b${ref}\\b`, 'g');
            // eslint-disable-next-line no-useless-escape
            sql = sql.replace(re, `\$\{${ref}\}`);
        });
    }
    else {
        try {
            type = (0, field_1.parseMetricType)(metric.calculation_method);
        }
        catch (e) {
            throw new errors_1.ParseError(`Cannot parse metric '${metric.unique_id}: type ${metric.calculation_method} is not a valid Lightdash metric type`);
        }
        sql = (0, field_1.defaultSql)(metric.name);
        if (metric.expression) {
            const isSingleColumnName = /^[a-zA-Z0-9_]+$/g.test(metric.expression);
            if (isSingleColumnName) {
                sql = (0, field_1.defaultSql)(metric.expression);
            }
            else {
                sql = metric.expression;
            }
        }
    }
    if (metric.filters && metric.filters.length > 0) {
        const filterSql = metric.filters
            .map((filter) => 
        // eslint-disable-next-line no-useless-escape
        `(\$\{TABLE\}.${filter.field} ${filter.operator} ${filter.value})`)
            .join(' AND ');
        sql = `CASE WHEN ${filterSql} THEN ${sql} ELSE NULL END`;
    }
    const groups = (0, dbt_1.convertToGroups)(metric.meta?.groups, metric.meta?.group_label);
    const spotlightVisibility = spotlightConfig.default_visibility;
    const spotlightCategories = (0, lightdashProjectConfig_1.getCategoriesFromResource)('metric', metric.name, spotlightConfig, Array.from(new Set([...(modelCategories || [])])));
    return {
        fieldType: field_1.FieldType.METRIC,
        type,
        name: metric.name,
        label: metric.label || (0, field_1.friendlyName)(metric.name),
        table: tableName,
        tableLabel,
        sql,
        description: metric.description,
        source: undefined,
        hidden: !!metric.meta?.hidden,
        round: metric.meta?.round,
        compact: metric.meta?.compact,
        format: metric.meta?.format,
        separator: metric.meta?.separator,
        groups,
        percentile: metric.meta?.percentile,
        showUnderlyingValues: metric.meta?.show_underlying_values,
        filters: (0, filterGrammar_1.parseFilters)(metric.meta?.filters),
        ...(metric.meta?.urls ? { urls: metric.meta.urls } : {}),
        ...(metric.meta && metric.meta.tags
            ? {
                tags: Array.isArray(metric.meta.tags)
                    ? metric.meta.tags
                    : [metric.meta.tags],
            }
            : {}),
        ...(0, lightdashProjectConfig_1.getSpotlightConfigurationForResource)({
            visibility: spotlightVisibility,
            categories: spotlightCategories,
            owner: metric.meta?.spotlight?.owner,
        }),
    };
};
function normalizePrimaryKey(primaryKey) {
    if (primaryKey) {
        return Array.isArray(primaryKey) ? primaryKey : [primaryKey];
    }
    return undefined;
}
function validateSets(dimensions, allMetrics, model, meta, allowPartialCompilation) {
    const warnings = [];
    const allFieldNames = new Set([
        ...Object.keys(dimensions),
        ...Object.keys(allMetrics),
    ]);
    if (!meta.sets)
        return warnings;
    Object.entries(meta.sets).forEach(([setName, setDef]) => {
        // Validate set name doesn't conflict with field names
        if (allFieldNames.has(setName)) {
            const errorMessage = `Set name "${setName}" in model "${model.name}" conflicts with an existing field name. Set names must be unique from dimension and metric names.`;
            if (allowPartialCompilation) {
                warnings.push({
                    type: explore_1.InlineErrorType.SET_VALIDATION_ERROR,
                    message: errorMessage,
                });
                return; // Skip this set
            }
            throw new errors_1.ParseError(errorMessage);
        }
        // Validate set definition structure
        if (!setDef.fields || !Array.isArray(setDef.fields)) {
            const errorMessage = `Set "${setName}" in model "${model.name}" must have a "fields" array`;
            if (allowPartialCompilation) {
                warnings.push({
                    type: explore_1.InlineErrorType.SET_VALIDATION_ERROR,
                    message: errorMessage,
                });
                return; // Skip this set
            }
            throw new errors_1.ParseError(errorMessage);
        }
        if (setDef.fields.length === 0) {
            const errorMessage = `Set "${setName}" in model "${model.name}" cannot be empty`;
            if (allowPartialCompilation) {
                warnings.push({
                    type: explore_1.InlineErrorType.SET_VALIDATION_ERROR,
                    message: errorMessage,
                });
                return; // Skip this set
            }
            throw new errors_1.ParseError(errorMessage);
        }
        // Validate field names don't have invalid characters
        setDef.fields.forEach((field) => {
            if (typeof field !== 'string') {
                const errorMessage = `Set "${setName}" in model "${model.name}" contains non-string field: ${field}`;
                if (allowPartialCompilation) {
                    warnings.push({
                        type: explore_1.InlineErrorType.SET_VALIDATION_ERROR,
                        message: errorMessage,
                    });
                    return; // Skip this field
                }
                throw new errors_1.ParseError(errorMessage);
            }
            // Allow field references ending with * (set references)
            // Allow field references starting with - (exclusions)
            const cleanField = field.replace(/^-/, '').replace(/\*$/, '');
            // Validate that the clean field name follows the lightdash variable pattern
            // (letters, numbers, underscores, and dots only)
            if (cleanField && !/^[a-zA-Z0-9_.]+$/.test(cleanField)) {
                const errorMessage = `Set "${setName}" in model "${model.name}" contains invalid field name "${field}". Field names must contain only letters, numbers, underscores, and dots.`;
                if (allowPartialCompilation) {
                    warnings.push({
                        type: explore_1.InlineErrorType.SET_VALIDATION_ERROR,
                        message: errorMessage,
                    });
                    return; // Skip this field
                }
                throw new errors_1.ParseError(errorMessage);
            }
            const isModelFieldName = !field.endsWith('*') && !field.startsWith('-');
            // Validate that regular field references exist in the model
            if (isModelFieldName) {
                const [joinName, fieldName] = field.includes('.')
                    ? field.split('.')
                    : [null, field];
                if (joinName) {
                    const joins = model.meta?.joins || [];
                    const allJoinNames = joins.map((j) => j.alias || j.join);
                    if (!allJoinNames.includes(joinName)) {
                        const errorMessage = `Set "${setName}" in model "${model.name}" references non-existent join model "${joinName}".`;
                        if (allowPartialCompilation) {
                            warnings.push({
                                type: explore_1.InlineErrorType.SET_VALIDATION_ERROR,
                                message: errorMessage,
                            });
                            return; // Skip this field
                        }
                        throw new errors_1.ParseError(errorMessage);
                    }
                }
                else if (!allFieldNames.has(fieldName)) {
                    const errorMessage = `Set "${setName}" in model "${model.name}" references non-existent model field "${field}". Fields must correspond to actual dimensions or metrics in the model.`;
                    if (allowPartialCompilation) {
                        warnings.push({
                            type: explore_1.InlineErrorType.SET_VALIDATION_ERROR,
                            message: errorMessage,
                        });
                        return; // Skip this field
                    }
                    throw new errors_1.ParseError(errorMessage);
                }
            }
            // Validate nesting level: if this set references another set,
            // we check up to 3 levels of nesting
            const checkNesting = (fields, depth) => {
                if (depth > 3) {
                    const errorMessage = `Set "${setName}" in model "${model.name}" exceeds the maximum nesting level of 3.`;
                    if (allowPartialCompilation) {
                        warnings.push({
                            type: explore_1.InlineErrorType.SET_VALIDATION_ERROR,
                            message: errorMessage,
                        });
                        return; // Stop checking nesting
                    }
                    throw new errors_1.ParseError(errorMessage);
                }
                fields.forEach((f) => {
                    if (typeof f === 'string' &&
                        f.endsWith('*') &&
                        !f.startsWith('-')) {
                        const referencedSetName = f.substring(0, f.length - 1);
                        const referencedSet = meta.sets?.[referencedSetName];
                        if (referencedSet) {
                            checkNesting(referencedSet.fields, depth + 1);
                        }
                    }
                });
            };
            if (!isModelFieldName && field.endsWith('*')) {
                const referencedSetName = field.substring(0, field.length - 1);
                const referencedSet = meta.sets?.[referencedSetName];
                if (referencedSet) {
                    checkNesting(referencedSet.fields, 2);
                }
            }
        });
    });
    return warnings;
}
const convertTable = (adapterType, model, dbtMetrics, spotlightConfig, startOfWeek, disableTimestampConversion, customGranularities, allowPartialCompilation) => {
    // Config block takes priority, then meta block
    const meta = (0, merge_1.default)({}, model.meta, model.config?.meta);
    const tableLabel = meta.label || (0, field_1.friendlyName)(model.name);
    const tableWarnings = [];
    const [dimensions, metrics] = Object.values(model.columns).reduce(([prevDimensions, prevMetrics], column, index) => {
        const dimension = convertDimension(index, adapterType, model, tableLabel, column, undefined, undefined, startOfWeek, undefined, disableTimestampConversion);
        // Config block takes priority, then meta block
        const columnMeta = (0, merge_1.default)({}, column.meta, column.config?.meta);
        const processIntervalDimension = (dim, overrideTimeIntervals) => {
            if (dim.isIntervalBase) {
                let allIntervals = [];
                if (!dim.isAdditionalDimension &&
                    columnMeta.dimension?.time_intervals &&
                    Array.isArray(columnMeta?.dimension.time_intervals)) {
                    allIntervals = columnMeta.dimension.time_intervals;
                }
                else if (dim.isAdditionalDimension &&
                    Array.isArray(overrideTimeIntervals)) {
                    allIntervals = overrideTimeIntervals;
                }
                else {
                    allIntervals = (0, timeFrames_1.getDefaultTimeFrames)(dim.type);
                }
                // Split into standard TimeFrames and custom granularity names
                const intervals = (0, timeFrames_1.validateTimeFrames)(allIntervals);
                const customIntervalNames = allIntervals.filter((v) => !(0, timeFrames_1.isTimeInterval)(v.toUpperCase()));
                const dimensionMeta = {
                    ...columnMeta.dimension,
                    type: dim.type,
                    label: dim.label,
                    groups: dim.groups,
                    sql: dim.sql,
                    description: dim.description,
                    hidden: dim.hidden,
                };
                // Generate standard interval dimensions
                const standardDims = intervals.reduce((acc, interval) => ({
                    ...acc,
                    [`${dim.name}_${interval.toLowerCase()}`]: convertDimension(index, adapterType, model, tableLabel, {
                        ...column,
                        ...('isAdditionalDimension' in dim &&
                            dim.isAdditionalDimension
                            ? {
                                name: dim.name,
                                meta: {
                                    dimension: dimensionMeta,
                                },
                                // In dbt 1.10+, config.meta takes precedence over meta
                                // so we must set config.meta.dimension to prevent
                                // the base dimension's properties from overwriting
                                config: {
                                    meta: {
                                        dimension: dimensionMeta,
                                    },
                                },
                            }
                            : {}),
                    }, undefined, interval, startOfWeek, 'isAdditionalDimension' in dim &&
                        dim.isAdditionalDimension, disableTimestampConversion),
                }), {});
                // Generate custom granularity dimensions
                const customDims = customIntervalNames.reduce((acc, customName) => {
                    const granularity = customGranularities?.[customName];
                    if (!granularity) {
                        tableWarnings.push({
                            type: explore_1.InlineErrorType.FIELD_ERROR,
                            message: `Unknown time interval "${customName}" on column "${dim.name}" in model "${model.name}". It is not a standard time frame or a custom granularity defined in lightdash.config.yml.`,
                        });
                        return acc;
                    }
                    const customSql = granularity.sql.replace(/\$\{COLUMN\}/g, () => dim.sql);
                    const customType = granularity.type || field_1.DimensionType.DATE;
                    const customDimName = `${dim.name}_${customName}`;
                    const groups = [...(dim.groups || [])];
                    if (!groups.includes(dim.label)) {
                        groups.push(dim.label);
                    }
                    return {
                        ...acc,
                        [customDimName]: {
                            index,
                            fieldType: field_1.FieldType.DIMENSION,
                            name: customDimName,
                            label: granularity.label,
                            sql: customSql,
                            table: model.name,
                            tableLabel,
                            type: customType,
                            description: dim.description,
                            source: undefined,
                            timeInterval: undefined,
                            timeIntervalBaseDimensionName: dim.name,
                            timeIntervalBaseDimensionType: dim.type,
                            customTimeInterval: customName,
                            hidden: dim.hidden,
                            format: undefined,
                            round: undefined,
                            compact: undefined,
                            requiredAttributes: dim.requiredAttributes,
                            anyAttributes: dim.anyAttributes,
                            groups,
                            isIntervalBase: false,
                            isAdditionalDimension: dim.isAdditionalDimension,
                            ...(dim.skipTimezoneConversion
                                ? { skipTimezoneConversion: true }
                                : {}),
                        },
                    };
                }, {});
                return { ...standardDims, ...customDims };
            }
            return {};
        };
        let extraDimensions = {
            ...processIntervalDimension(dimension, undefined),
        };
        extraDimensions = Object.entries(columnMeta.additional_dimensions || {}).reduce((acc, [subDimensionName, subDimension]) => {
            const additionalDim = convertDimension(index, adapterType, model, tableLabel, {
                ...column,
                name: subDimensionName,
                meta: {
                    dimension: subDimension,
                },
                config: {
                    meta: {
                        dimension: subDimension,
                    },
                },
            }, undefined, undefined, startOfWeek, true, disableTimestampConversion);
            return {
                ...acc,
                // When the additional dim is interval AND the base dimension is a interval base then we want to compute all additional dims with the parent intervals otherwise just set the additional dim
                [subDimensionName]: additionalDim,
                ...processIntervalDimension(additionalDim, subDimension.time_intervals),
            };
        }, extraDimensions);
        const columnMetrics = Object.fromEntries(Object.entries(columnMeta.metrics || {}).map(([name, metric]) => [
            name,
            (0, dbt_1.convertColumnMetric)({
                modelName: model.name,
                dimensionName: dimension.name,
                dimensionSql: dimension.sql,
                dimensionType: dimension.type,
                dimensionTimeInterval: dimension.timeInterval,
                name,
                metric,
                tableLabel,
                requiredAttributes: dimension.requiredAttributes, // TODO Join dimension required_attributes with metric required_attributes
                anyAttributes: dimension.anyAttributes, // TODO Join dimension any_attributes with metric any_attributes
                spotlightConfig: {
                    ...spotlightConfig,
                    default_visibility: meta.spotlight?.visibility ??
                        spotlightConfig.default_visibility,
                },
                modelCategories: meta.spotlight?.categories,
                modelOwner: meta.spotlight?.owner ?? meta.owner,
                defaultShowUnderlyingValues: meta.default_show_underlying_values,
            }),
        ]));
        return [
            {
                ...prevDimensions,
                [column.name]: dimension,
                ...extraDimensions,
            },
            { ...prevMetrics, ...columnMetrics },
        ];
    }, [{}, {}]);
    const modelMetrics = Object.fromEntries(Object.entries(meta.metrics || {}).map(([name, metric]) => [
        name,
        (0, dbt_1.convertModelMetric)({
            modelName: model.name,
            name,
            metric,
            tableLabel,
            spotlightConfig: {
                ...spotlightConfig,
                default_visibility: meta.spotlight?.visibility ??
                    spotlightConfig.default_visibility,
            },
            modelCategories: meta.spotlight?.categories,
            modelOwner: meta.spotlight?.owner ?? meta.owner,
            defaultShowUnderlyingValues: meta.default_show_underlying_values,
        }),
    ]));
    const convertedDbtMetrics = Object.fromEntries(dbtMetrics.map((metric) => [
        metric.name,
        convertDbtMetricToLightdashMetric(metric, model.name, tableLabel, {
            ...spotlightConfig,
            default_visibility: meta.spotlight?.visibility ??
                spotlightConfig.default_visibility,
        }, meta.spotlight?.categories),
    ]));
    const allMetrics = Object.values({
        ...convertedDbtMetrics,
        ...modelMetrics,
        ...metrics,
    }).reduce((acc, metric, index) => ({
        ...acc,
        [metric.name]: { ...metric, index },
    }), {});
    const duplicatedNames = Object.keys(allMetrics).filter((metric) => Object.keys(dimensions).includes(metric));
    if (duplicatedNames.length > 0) {
        duplicatedNames.forEach((name) => {
            delete allMetrics[name];
            tableWarnings.push({
                type: explore_1.InlineErrorType.DUPLICATE_FIELD_NAME,
                message: `Skipped metric "${name}" because a dimension with the same name exists. Dimensions take priority.`,
            });
        });
    }
    const groupDetails = {};
    if (meta.group_details) {
        Object.entries(meta.group_details).forEach(([key, data]) => {
            groupDetails[key] = {
                label: data.label,
                description: data.description,
            };
        });
    }
    if (meta.sets) {
        const warnings = validateSets(dimensions, allMetrics, model, meta, allowPartialCompilation);
        tableWarnings.push(...warnings);
    }
    const sqlTable = meta.sql_from || model.relation_name;
    if (sqlTable === null || sqlTable === undefined || sqlTable === '') {
        throw new Error(`Model "${model.name}" is missing a table reference.`);
    }
    return {
        name: model.name,
        label: tableLabel,
        database: model.database,
        schema: model.schema,
        sqlTable,
        description: model.description || `${model.name} table`,
        dimensions,
        metrics: allMetrics,
        orderFieldsBy: meta.order_fields_by &&
            Object.values(table_1.OrderFieldsByStrategy).includes(meta.order_fields_by.toUpperCase())
            ? meta.order_fields_by.toUpperCase()
            : table_1.OrderFieldsByStrategy.LABEL,
        groupLabel: meta.group_label,
        primaryKey: normalizePrimaryKey(meta.primary_key),
        sqlWhere: meta.sql_filter || meta.sql_where,
        requiredFilters: (0, filterGrammar_1.parseModelRequiredFilters)({
            requiredFilters: meta.required_filters,
            defaultFilters: meta.default_filters,
        }),
        requiredAttributes: meta.required_attributes,
        anyAttributes: meta.any_attributes,
        groupDetails,
        ...(meta.default_time_dimension
            ? {
                defaultTimeDimension: {
                    field: meta.default_time_dimension.field,
                    interval: meta.default_time_dimension.interval,
                },
            }
            : {}),
        ...(meta.default_show_underlying_values
            ? {
                defaultShowUnderlyingValues: meta.default_show_underlying_values,
            }
            : {}),
        ...(meta.ai_hint ? { aiHint: (0, dbt_1.convertToAiHints)(meta.ai_hint) } : {}),
        ...(meta.parameters ? { parameters: meta.parameters } : {}),
        ...(meta.sets ? { sets: meta.sets } : {}),
        ...(tableWarnings.length > 0 ? { warnings: tableWarnings } : {}),
        ...(model.package_name ? { dbtPackageName: model.package_name } : {}),
        ...(model.patch_path
            ? { ymlPath: (0, dbt_1.patchPathParts)(model.patch_path).path }
            : {}),
        ...(model.path ? { sqlPath: model.path } : {}),
    };
};
exports.convertTable = convertTable;
const translateDbtModelsToTableLineage = (models) => {
    const graph = (0, dbt_1.buildModelGraph)(models);
    return models.reduce((previousValue, currentValue) => ({
        ...previousValue,
        [currentValue.name]: {
            lineageGraph: generateTableLineage(currentValue, graph),
        },
    }), {});
};
const modelCanUseMetric = (metricName, modelName, metrics) => {
    const metric = metrics.find((m) => m.name === metricName);
    if (!metric) {
        return false;
    }
    const modelRef = metric?.refs?.[0];
    if (modelRef) {
        const modelRefName = (0, dbt_1.isV9MetricRef)(modelRef)
            ? modelRef.name
            : modelRef[0];
        if (modelRefName === modelName) {
            return true;
        }
    }
    if (metric.calculation_method === 'derived') {
        const referencedMetrics = (metric.metrics || []).map((m) => m[0]);
        return referencedMetrics.every((m) => modelCanUseMetric(m, modelName, metrics));
    }
    return false;
};
const convertExplores = async (models, loadSources, adapterType, metrics, warehouseSqlBuilder, lightdashProjectConfig, options) => {
    const { disableTimestampConversion, allowPartialCompilation, postProcessors, } = options ?? {};
    const tableLineage = translateDbtModelsToTableLineage(models);
    const [tables, exploreErrors] = models.reduce(([accTables, accErrors], model) => {
        // Config block takes priority, then meta block
        const meta = (0, merge_1.default)({}, model.meta, model.config?.meta);
        // model.config.tags has type string[] | string | undefined - normalise it to string[]
        const configTags = typeof model.config?.tags === 'string'
            ? [model.config.tags]
            : model.config?.tags;
        // model.config.tags takes priority over model.tags - if config tags is an empty list, we'll use model tags
        const tags = configTags && configTags.length > 0 ? configTags : model.tags;
        // If there are any errors compiling the table return an ExploreError
        try {
            // base dimensions and metrics
            // TODO: remove old metrics handling
            const tableMetrics = metrics.filter((metric) => modelCanUseMetric(metric.name, model.name, metrics));
            const table = (0, exports.convertTable)(adapterType, model, tableMetrics, lightdashProjectConfig.spotlight, warehouseSqlBuilder.getStartOfWeek(), disableTimestampConversion, lightdashProjectConfig.custom_granularities, allowPartialCompilation);
            // add lineage
            const tableWithLineage = {
                ...table,
                ...tableLineage[model.name],
            };
            return [[...accTables, tableWithLineage], accErrors];
        }
        catch (e) {
            const exploreError = {
                name: model.name,
                label: meta.label || (0, field_1.friendlyName)(model.name),
                tags,
                groupLabel: meta.group_label,
                ...(meta.groups && meta.groups.length > 0
                    ? { groups: meta.groups }
                    : {}),
                errors: [
                    {
                        type: e instanceof errors_1.ParseError
                            ? explore_1.InlineErrorType.METADATA_PARSE_ERROR
                            : explore_1.InlineErrorType.NO_DIMENSIONS_FOUND,
                        message: e instanceof Error
                            ? e.message
                            : `Could not convert dbt model: "${model.name}" in to a Lightdash explore`,
                    },
                ],
            };
            return [accTables, [...accErrors, exploreError]];
        }
    }, [[], []]);
    const tableLookup = tables.reduce((prev, table) => ({ ...prev, [table.name]: table }), {});
    const validModels = models.filter((model) => tableLookup[model.name] !== undefined &&
        // Seeds are compiled as tables (for join resolution) but should
        // not generate standalone explores — they're join targets only.
        model.resource_type !== 'seed');
    const exploreCompiler = new exploreCompiler_1.ExploreCompiler(warehouseSqlBuilder, {
        allowPartialCompilation,
    });
    const explores = validModels.reduce((acc, model) => {
        // Config block takes priority, then meta block
        const meta = (0, merge_1.default)({}, model.meta, model.config?.meta);
        const configTags = typeof model.config?.tags === 'string'
            ? [model.config.tags]
            : model.config?.tags;
        const tags = configTags && configTags.length > 0 ? configTags : model.tags;
        // Create an array of explores to generate: base explore + any additional explores
        const exploresToCreate = [
            {
                name: model.name,
                label: meta.label || (0, field_1.friendlyName)(model.name),
                groupLabel: meta.group_label,
                ...(meta.groups && meta.groups.length > 0
                    ? { groups: meta.groups }
                    : {}),
                joins: meta?.joins || [],
                description: meta.description,
                caseSensitive: meta.case_sensitive,
                tables: tableLookup,
            },
            ...(meta.explores
                ? Object.entries(meta.explores).map(([exploreName, exploreConfig]) => {
                    const baseTable = tableLookup[model.name];
                    const baseTableLabel = meta.label || (0, field_1.friendlyName)(model.name);
                    // Convert explore-scoped additional dimensions
                    let exploreScopedDimensions = {};
                    if (exploreConfig.additional_dimensions) {
                        const existingDimensionCount = Object.keys(baseTable.dimensions).length;
                        Object.entries(exploreConfig.additional_dimensions).forEach(([dimName, dimConfig], dimIndex) => {
                            const convertedDims = convertExploreScopedDimension(existingDimensionCount + dimIndex, model.name, baseTableLabel, dimName, dimConfig, adapterType, warehouseSqlBuilder.getStartOfWeek());
                            exploreScopedDimensions = {
                                ...exploreScopedDimensions,
                                ...convertedDims,
                            };
                        });
                    }
                    return {
                        name: exploreName,
                        label: exploreConfig.label ||
                            (0, field_1.friendlyName)(exploreName),
                        groupLabel: exploreConfig.group_label || meta.group_label,
                        ...((exploreConfig.groups &&
                            exploreConfig.groups.length > 0) ||
                            (meta.groups && meta.groups.length > 0)
                            ? {
                                groups: exploreConfig.groups || meta.groups,
                            }
                            : {}),
                        // Inherit joins from base model if not specified in explore config
                        joins: exploreConfig.joins || meta?.joins || [],
                        description: exploreConfig.description,
                        caseSensitive: exploreConfig.case_sensitive,
                        tables: {
                            ...tableLookup,
                            // Override the base table with required filters and explore-scoped dimensions
                            [model.name]: {
                                ...baseTable,
                                sqlWhere: exploreConfig.sql_filter ||
                                    exploreConfig.sql_where ||
                                    baseTable.sqlWhere,
                                requiredFilters: (0, filterGrammar_1.parseModelRequiredFilters)({
                                    requiredFilters: exploreConfig.required_filters,
                                    defaultFilters: exploreConfig.default_filters,
                                }),
                                // Merge explore-scoped dimensions with existing dimensions
                                dimensions: {
                                    ...baseTable.dimensions,
                                    ...exploreScopedDimensions,
                                },
                            },
                        },
                    };
                })
                : []),
        ];
        // Multiple explores can be created from a single model. The base explore + additional explores
        // Properties created from `model` are the same across all explores. e.g. all explores will have the same base table & warehouse
        // Properties created from `exploreToCreate` are specific to each explore. e.g. each explore can have a different name, label & joins
        const compiledExplores = exploresToCreate.map((exploreToCreate) => {
            try {
                return exploreCompiler.compileExplore({
                    name: exploreToCreate.name,
                    label: exploreToCreate.label,
                    tags: tags || [],
                    baseTable: model.name,
                    groupLabel: exploreToCreate.groupLabel,
                    ...(exploreToCreate.groups &&
                        exploreToCreate.groups.length > 0
                        ? { groups: exploreToCreate.groups }
                        : {}),
                    caseSensitive: exploreToCreate.caseSensitive,
                    joinedTables: exploreToCreate.joins.map((join) => ({
                        table: join.join,
                        sqlOn: join.sql_on,
                        type: join.type,
                        alias: join.alias,
                        label: join.label,
                        fields: join.fields,
                        hidden: join.hidden,
                        always: join.always,
                        relationship: join.relationship,
                        description: join.description,
                    })),
                    tables: exploreToCreate.tables,
                    targetDatabase: adapterType,
                    warehouse: model.config?.snowflake_warehouse,
                    databricksCompute: model.config?.databricks_compute,
                    ymlPath: model.patch_path
                        ? (0, dbt_1.patchPathParts)(model.patch_path).path
                        : undefined,
                    sqlPath: model.path,
                    spotlightConfig: lightdashProjectConfig.spotlight,
                    ...(meta.ai_hint
                        ? { aiHint: (0, dbt_1.convertToAiHints)(meta.ai_hint) }
                        : {}),
                    meta: {
                        ...meta,
                        // Override description for additional explores
                        ...(exploreToCreate.description !== undefined
                            ? { description: exploreToCreate.description }
                            : {}),
                    },
                    projectParameters: lightdashProjectConfig.parameters,
                    projectDefaults: lightdashProjectConfig.defaults,
                });
            }
            catch (e) {
                return {
                    name: exploreToCreate.name,
                    label: exploreToCreate.label,
                    groupLabel: exploreToCreate.groupLabel,
                    ...(exploreToCreate.groups &&
                        exploreToCreate.groups.length > 0
                        ? { groups: exploreToCreate.groups }
                        : {}),
                    errors: [
                        {
                            // TODO improve parsing of error type
                            type: e instanceof errors_1.ParseError ||
                                e instanceof errors_1.CompileError
                                ? explore_1.InlineErrorType.METADATA_PARSE_ERROR
                                : explore_1.InlineErrorType.NO_DIMENSIONS_FOUND,
                            message: e instanceof Error
                                ? e.message
                                : `Could not convert ${exploreToCreate.name === model.name
                                    ? 'dbt model'
                                    : 'additional explore'}: "${exploreToCreate.name}" ${exploreToCreate.name !== model.name
                                    ? `from model "${model.name}"`
                                    : 'is not a valid model'}`,
                        },
                    ],
                };
            }
        });
        // Split compiled explores into successes and errors,
        // then run post-processors over successful explores only
        const [compileErrors, successfulExplores] = (0, partition_1.default)(compiledExplores, explore_1.isExploreError);
        const postProcessorContext = {
            model,
            meta,
        };
        const postProcessedExplores = (postProcessors ?? []).reduce((currentExplores, processor) => {
            const [errors, successes] = (0, partition_1.default)(currentExplores, explore_1.isExploreError);
            return [...errors, ...processor(successes, postProcessorContext)];
        }, successfulExplores);
        return [...acc, ...compileErrors, ...postProcessedExplores];
    }, []);
    return [...explores, ...exploreErrors];
};
exports.convertExplores = convertExplores;
const attachTypesToModels = (models, warehouseCatalog, throwOnMissingCatalogEntry = true, caseSensitiveMatching = true) => {
    // Check that all models appear in the warehouse
    models.forEach(({ database, schema, name }) => {
        const databaseMatch = Object.keys(warehouseCatalog).find((db) => caseSensitiveMatching
            ? db === database
            : db.toLowerCase() === database.toLowerCase());
        const schemaMatch = databaseMatch &&
            Object.keys(warehouseCatalog[databaseMatch]).find((s) => caseSensitiveMatching
                ? s === schema
                : s.toLowerCase() === schema.toLowerCase());
        const tableMatch = databaseMatch &&
            schemaMatch &&
            Object.keys(warehouseCatalog[databaseMatch][schemaMatch]).find((t) => caseSensitiveMatching
                ? t === name
                : t.toLowerCase() === name.toLowerCase());
        if (!tableMatch && throwOnMissingCatalogEntry) {
            throw new errors_1.MissingCatalogEntryError(`Model "${name}" was expected in your target warehouse at "${database}.${schema}.${name}". Does the table exist in your target data warehouse?`, {});
        }
    });
    const getType = ({ database, schema, name, alias }, columnName) => {
        const tableName = alias || name;
        const databaseMatch = Object.keys(warehouseCatalog).find((db) => caseSensitiveMatching
            ? db === database
            : db.toLowerCase() === database.toLowerCase());
        const schemaMatch = databaseMatch &&
            Object.keys(warehouseCatalog[databaseMatch]).find((s) => caseSensitiveMatching
                ? s === schema
                : s.toLowerCase() === schema.toLowerCase());
        const tableMatch = databaseMatch &&
            schemaMatch &&
            Object.keys(warehouseCatalog[databaseMatch][schemaMatch]).find((t) => caseSensitiveMatching
                ? t === tableName
                : t.toLowerCase() === tableName.toLowerCase());
        const columnMatch = databaseMatch &&
            schemaMatch &&
            tableMatch &&
            Object.keys(warehouseCatalog[databaseMatch][schemaMatch][tableMatch]).find((c) => caseSensitiveMatching
                ? c === columnName
                : c.toLowerCase() === columnName.toLowerCase());
        if (databaseMatch && schemaMatch && tableMatch && columnMatch) {
            return warehouseCatalog[databaseMatch][schemaMatch][tableMatch][columnMatch];
        }
        if (throwOnMissingCatalogEntry) {
            throw new errors_1.MissingCatalogEntryError(`Column "${columnName}" from model "${tableName}" does not exist.\n "${tableName}.${columnName}" was not found in your target warehouse at ${database}.${schema}.${tableName}. Try rerunning dbt to update your warehouse.`, {});
        }
        return undefined;
    };
    // Update the dbt models with type info
    return models.map((model) => ({
        ...model,
        columns: Object.fromEntries(Object.entries(model.columns).map(([column_name, column]) => [
            column_name,
            { ...column, data_type: getType(model, column_name) },
        ])),
    }));
};
exports.attachTypesToModels = attachTypesToModels;
const getSchemaStructureFromDbtModels = (dbtModels) => dbtModels.map(({ database, schema, name, alias }) => ({
    database,
    schema,
    table: alias || name,
}));
exports.getSchemaStructureFromDbtModels = getSchemaStructureFromDbtModels;
//# sourceMappingURL=translator.js.map