"use strict";
/*
 * Copyright © 2020 Atomist, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
Object.defineProperty(exports, "__esModule", { value: true });
exports.k8sContainerFulfiller = exports.K8sContainerFulfillerName = exports.executeK8sJob = exports.scheduleK8sJob = exports.k8sFulfillmentCallback = exports.k8sSkillContainerScheduler = exports.k8sContainerScheduler = void 0;
const poll_1 = require("@atomist/automation-client/lib/internal/util/poll");
const string_1 = require("@atomist/automation-client/lib/internal/util/string");
const GitCommandGitProject_1 = require("@atomist/automation-client/lib/project/git/GitCommandGitProject");
const k8s = require("@kubernetes/client-node");
const fs = require("fs-extra");
const stringify = require("json-stringify-safe");
const _ = require("lodash");
const os = require("os");
const path = require("path");
const stream_1 = require("stream");
const minimalClone_1 = require("../../api-helper/goal/minimalClone");
const sdmGoal_1 = require("../../api-helper/goal/sdmGoal");
const GoalInvocation_1 = require("../../api/goal/GoalInvocation");
const GoalWithFulfillment_1 = require("../../api/goal/GoalWithFulfillment");
const ServiceRegistration_1 = require("../../api/registration/ServiceRegistration");
const goalCaching_1 = require("../../core/goal/cache/goalCaching");
const container_1 = require("../../core/goal/container/container");
const provider_1 = require("../../core/goal/container/provider");
const util_1 = require("../../core/goal/container/util");
const array_1 = require("../../core/util/misc/array");
const types_1 = require("../../typings/types");
const config_1 = require("./kubernetes/config");
const KubernetesGoalScheduler_1 = require("./scheduler/KubernetesGoalScheduler");
const service_1 = require("./scheduler/service");
const error_1 = require("./support/error");
/**
 * Container scheduler to use when running in Kubernetes.
 */
const k8sContainerScheduler = (goal, registration) => {
    goal.addFulfillment(Object.assign({ goalExecutor: executeK8sJob() }, registration));
    goal.addFulfillmentCallback({
        goal,
        callback: k8sFulfillmentCallback(goal, registration),
    });
};
exports.k8sContainerScheduler = k8sContainerScheduler;
/**
 * Container scheduler to use when running in Google Cloud Functions.
 */
const k8sSkillContainerScheduler = (goal, registration) => {
    goal.addFulfillment(Object.assign({ goalExecutor: executeK8sJob() }, registration));
};
exports.k8sSkillContainerScheduler = k8sSkillContainerScheduler;
/**
 * Add Kubernetes job scheduling information to SDM goal event data
 * for use by the [[KubernetesGoalScheduler]].
 */
function k8sFulfillmentCallback(goal, registration) {
    // tslint:disable-next-line:cyclomatic-complexity
    return async (goalEvent, repoContext) => {
        let spec = _.cloneDeep(registration);
        if (registration.callback) {
            spec = await repoContext.configuration.sdm.projectLoader.doWithProject(Object.assign(Object.assign({}, repoContext), { readOnly: true, cloneOptions: minimalClone_1.minimalClone(goalEvent.push, { detachHead: true }) }), async (p) => {
                return Object.assign(Object.assign({}, spec), ((await registration.callback(_.cloneDeep(registration), p, goal, goalEvent, repoContext)) ||
                    {}));
            });
        }
        if (!spec.containers || spec.containers.length < 1) {
            throw new Error("No containers defined in K8sGoalContainerSpec");
        }
        // Preserve the container registration in the goal data before it gets munged with internals
        let data = sdmGoal_1.goalData(goalEvent);
        let newData = {};
        delete spec.callback;
        _.set(newData, container_1.ContainerRegistrationGoalDataKey, spec);
        goalEvent.data = JSON.stringify(_.merge(data, newData));
        if (spec.containers[0].workingDir === "") {
            delete spec.containers[0].workingDir;
        }
        else if (!spec.containers[0].workingDir) {
            spec.containers[0].workingDir = container_1.ContainerProjectHome;
        }
        const goalSchedulers = array_1.toArray(repoContext.configuration.sdm.goalScheduler) || [];
        const k8sScheduler = goalSchedulers.find(gs => gs instanceof KubernetesGoalScheduler_1.KubernetesGoalScheduler);
        if (!k8sScheduler) {
            throw new Error("Failed to find KubernetesGoalScheduler in goal schedulers");
        }
        if (!k8sScheduler.podSpec) {
            throw new Error("KubernetesGoalScheduler has no podSpec defined");
        }
        const containerEnvs = await util_1.containerEnvVars(goalEvent, repoContext);
        const projectVolume = `project-${string_1.guid().split("-")[0]}`;
        const inputVolume = `input-${string_1.guid().split("-")[0]}`;
        const outputVolume = `output-${string_1.guid().split("-")[0]}`;
        const ioVolumes = [
            {
                name: projectVolume,
                emptyDir: {},
            },
            {
                name: inputVolume,
                emptyDir: {},
            },
            {
                name: outputVolume,
                emptyDir: {},
            },
        ];
        const ioVolumeMounts = [
            {
                mountPath: container_1.ContainerProjectHome,
                name: projectVolume,
            },
            {
                mountPath: container_1.ContainerInput,
                name: inputVolume,
            },
            {
                mountPath: container_1.ContainerOutput,
                name: outputVolume,
            },
        ];
        const copyContainer = _.cloneDeep(k8sScheduler.podSpec.containers[0]);
        delete copyContainer.lifecycle;
        delete copyContainer.livenessProbe;
        delete copyContainer.readinessProbe;
        copyContainer.name = `container-goal-init-${string_1.guid().split("-")[0]}`;
        copyContainer.env = [
            ...(copyContainer.env || []),
            ...KubernetesGoalScheduler_1.k8sJobEnv(k8sScheduler.podSpec, goalEvent, repoContext.context),
            ...containerEnvs,
            {
                name: "ATOMIST_ISOLATED_GOAL_INIT",
                value: "true",
            },
            {
                name: "ATOMIST_CONFIG",
                value: JSON.stringify({
                    cluster: {
                        enabled: false,
                    },
                    ws: {
                        enabled: false,
                    },
                }),
            },
        ];
        spec.initContainers = spec.initContainers || [];
        const parameters = JSON.parse(goalEvent.parameters || "{}");
        const secrets = await provider_1.prepareSecrets(_.merge({}, registration.containers[0], parameters["@atomist/sdm/secrets"] || {}), repoContext);
        delete spec.containers[0].secrets;
        [...spec.containers, ...spec.initContainers].forEach(c => {
            c.env = [...(secrets.env || []), ...containerEnvs, ...(c.env || [])];
        });
        if (!!(secrets === null || secrets === void 0 ? void 0 : secrets.files)) {
            for (const file of secrets.files) {
                const fileName = path.basename(file.mountPath);
                const dirname = path.dirname(file.mountPath);
                let secretName = `secret-${string_1.guid().split("-")[0]}`;
                const vm = (copyContainer.volumeMounts || []).find(m => m.mountPath === dirname);
                if (!!vm) {
                    secretName = vm.name;
                }
                else {
                    copyContainer.volumeMounts = [
                        ...(copyContainer.volumeMounts || []),
                        {
                            mountPath: dirname,
                            name: secretName,
                        },
                    ];
                    spec.volumes = [
                        ...(spec.volumes || []),
                        {
                            name: secretName,
                            emptyDir: {},
                        },
                    ];
                }
                [...spec.containers, ...spec.initContainers].forEach((c) => {
                    c.volumeMounts = [
                        ...(c.volumeMounts || []),
                        {
                            mountPath: file.mountPath,
                            name: secretName,
                            subPath: fileName,
                        },
                    ];
                });
            }
        }
        spec.initContainers = [copyContainer, ...spec.initContainers];
        const serviceSpec = {
            type: service_1.K8sServiceRegistrationType.K8sService,
            spec: {
                container: spec.containers,
                initContainer: spec.initContainers,
                volume: [...ioVolumes, ...(spec.volumes || [])],
                volumeMount: ioVolumeMounts,
            },
        };
        // Store k8s service registration in goal data
        data = sdmGoal_1.goalData(goalEvent);
        newData = {};
        _.set(newData, `${ServiceRegistration_1.ServiceRegistrationGoalDataKey}.${registration.name}`, serviceSpec);
        goalEvent.data = JSON.stringify(_.merge(data, newData));
        return goalEvent;
    };
}
exports.k8sFulfillmentCallback = k8sFulfillmentCallback;
/**
 * Get container registration from goal event data, use
 * [[k8sFulfillmentcallback]] to get a goal event schedulable by a
 * [[KubernetesGoalScheduler]], then schedule the goal using that
 * scheduler.
 */
const scheduleK8sJob = async (gi) => {
    const { goalEvent } = gi;
    const { uniqueName } = goalEvent;
    const data = sdmGoal_1.goalData(goalEvent);
    const containerReg = data["@atomist/sdm/container"];
    if (!containerReg) {
        throw new Error(`Goal ${uniqueName} event data has no container spec: ${goalEvent.data}`);
    }
    const goalSchedulers = array_1.toArray(gi.configuration.sdm.goalScheduler) || [];
    const k8sScheduler = goalSchedulers.find(gs => gs instanceof KubernetesGoalScheduler_1.KubernetesGoalScheduler);
    if (!k8sScheduler) {
        throw new Error(`Failed to find KubernetesGoalScheduler in goal schedulers: ${stringify(goalSchedulers)}`);
    }
    // the k8sFulfillmentCallback may already have been called, so wipe it out
    delete data[ServiceRegistration_1.ServiceRegistrationGoalDataKey];
    goalEvent.data = JSON.stringify(data);
    try {
        const schedulableGoalEvent = await k8sFulfillmentCallback(gi.goal, containerReg)(goalEvent, gi);
        const scheduleResult = await k8sScheduler.schedule(Object.assign(Object.assign({}, gi), { goalEvent: schedulableGoalEvent }));
        if (scheduleResult.code) {
            return Object.assign(Object.assign({}, scheduleResult), { message: `Failed to schedule container goal ${uniqueName}: ${scheduleResult.message}` });
        }
        schedulableGoalEvent.state = types_1.SdmGoalState.in_process;
        return schedulableGoalEvent;
    }
    catch (e) {
        const message = `Failed to schedule container goal ${uniqueName} as Kubernetes job: ${e.message}`;
        gi.progressLog.write(message);
        return { code: 1, message };
    }
};
exports.scheduleK8sJob = scheduleK8sJob;
/**
 * Wait for first container to exit and stream its logs to the
 * progress log.
 */
function executeK8sJob() {
    // tslint:disable-next-line:cyclomatic-complexity
    return async (gi) => {
        const { goalEvent, progressLog, configuration, id, credentials } = gi;
        const projectDir = process.env.ATOMIST_PROJECT_DIR || container_1.ContainerProjectHome;
        const inputDir = process.env.ATOMIST_INPUT_DIR || container_1.ContainerInput;
        const outputDir = process.env.ATOMIST_OUTPUT_DIR || container_1.ContainerOutput;
        const data = sdmGoal_1.goalData(goalEvent);
        if (!data[container_1.ContainerRegistrationGoalDataKey]) {
            throw new Error("Failed to read k8s ContainerRegistration from goal data");
        }
        if (!data[container_1.ContainerRegistrationGoalDataKey]) {
            throw new Error(`Goal ${gi.goal.uniqueName} has no Kubernetes container registration: ${gi.goalEvent.data}`);
        }
        const registration = data[container_1.ContainerRegistrationGoalDataKey];
        if (process.env.ATOMIST_ISOLATED_GOAL_INIT === "true") {
            return configuration.sdm.projectLoader.doWithProject(Object.assign(Object.assign({}, gi), { readOnly: false, cloneDir: projectDir, cloneOptions: minimalClone_1.minimalClone(goalEvent.push, { detachHead: true }) }), async () => {
                try {
                    await util_1.prepareInputAndOutput(inputDir, outputDir, gi);
                }
                catch (e) {
                    const message = `Failed to prepare input and output for goal ${goalEvent.name}: ${e.message}`;
                    progressLog.write(message);
                    return { code: 1, message };
                }
                const secrets = await provider_1.prepareSecrets(_.merge({}, registration.containers[0], (gi.parameters || {})["@atomist/sdm/secrets"] || {}), gi);
                if (!!(secrets === null || secrets === void 0 ? void 0 : secrets.files)) {
                    for (const file of secrets.files) {
                        await fs.writeFile(file.mountPath, file.value);
                    }
                }
                goalEvent.state = types_1.SdmGoalState.in_process;
                return goalEvent;
            });
        }
        let containerName = _.get(registration, "containers[0].name");
        if (!containerName) {
            const msg = `Failed to get main container name from goal registration: ${stringify(registration)}`;
            progressLog.write(msg);
            let svcSpec;
            try {
                svcSpec = _.get(data, `${ServiceRegistration_1.ServiceRegistrationGoalDataKey}.${registration.name}.spec`);
            }
            catch (e) {
                const message = `Failed to parse Kubernetes spec from goal data '${goalEvent.data}': ${e.message}`;
                progressLog.write(message);
                return { code: 1, message };
            }
            containerName = _.get(svcSpec, "container[1].name");
            if (!containerName) {
                const message = `Failed to get main container name from either goal registration or data: '${goalEvent.data}'`;
                progressLog.write(message);
                return { code: 1, message };
            }
        }
        const ns = await KubernetesGoalScheduler_1.readNamespace();
        const podName = os.hostname();
        let kc;
        try {
            kc = config_1.loadKubeConfig();
        }
        catch (e) {
            const message = `Failed to load Kubernetes configuration: ${e.message}`;
            progressLog.write(message);
            return { code: 1, message };
        }
        const container = {
            config: kc,
            name: containerName,
            pod: podName,
            ns,
            log: progressLog,
        };
        try {
            await containerStarted(container);
        }
        catch (e) {
            const message = `Failed to determine if container started: ${e.message}`;
            progressLog.write(message);
            return { code: 1, message };
        }
        const status = { code: 0, message: `Container '${containerName}' completed successfully` };
        try {
            const timeout = sdmGoal_1.sdmGoalTimeout(configuration);
            const podStatus = await containerWatch(container, timeout);
            progressLog.write(`Container '${containerName}' exited: ${stringify(podStatus)}`);
        }
        catch (e) {
            const message = `Container '${containerName}' failed: ${e.message}`;
            progressLog.write(message);
            status.code++;
            status.message = message;
        }
        const outputFile = path.join(outputDir, "result.json");
        let outputResult;
        if (status.code === 0 && (await fs.pathExists(outputFile))) {
            try {
                outputResult = await util_1.processResult(await fs.readJson(outputFile), gi);
            }
            catch (e) {
                const message = `Failed to read output from container: ${e.message}`;
                progressLog.write(message);
                status.code++;
                status.message += ` but f${message.slice(1)}`;
            }
        }
        const cacheEntriesToPut = [
            ...(registration.output || []),
            ...((gi.parameters || {})[goalCaching_1.CacheOutputGoalDataKey] || []),
        ];
        if (cacheEntriesToPut.length > 0) {
            try {
                const project = GitCommandGitProject_1.GitCommandGitProject.fromBaseDir(id, projectDir, credentials, async () => { });
                const cp = goalCaching_1.cachePut({
                    entries: cacheEntriesToPut.map(e => {
                        // Prevent the type on the entry to get passed along when goal actually failed
                        if (status.code !== 0) {
                            return {
                                classifier: e.classifier,
                                pattern: e.pattern,
                            };
                        }
                        else {
                            return e;
                        }
                    }),
                });
                await cp.listener(project, gi, GoalInvocation_1.GoalProjectListenerEvent.after);
            }
            catch (e) {
                const message = `Failed to put cache output from container: ${e.message}`;
                progressLog.write(message);
                status.code++;
                status.message += ` but f${message.slice(1)}`;
            }
        }
        return outputResult || status;
    };
}
exports.executeK8sJob = executeK8sJob;
/**
 * If running as isolated goal, use [[executeK8sJob]] to execute the
 * goal.  Otherwise, schedule the goal execution as a Kubernetes job
 * using [[scheduleK8sJob]].
 */
const containerExecutor = gi => process.env.ATOMIST_ISOLATED_GOAL ? executeK8sJob()(gi) : exports.scheduleK8sJob(gi);
/**
 * Restore cache input entries before fulfilling goal.
 */
const containerFulfillerCacheRestore = {
    name: "cache restore",
    events: [GoalInvocation_1.GoalProjectListenerEvent.before],
    listener: async (project, gi) => {
        const data = sdmGoal_1.goalData(gi.goalEvent);
        if (!data[container_1.ContainerRegistrationGoalDataKey]) {
            throw new Error(`Goal ${gi.goal.uniqueName} has no Kubernetes container registration: ${gi.goalEvent.data}`);
        }
        const registration = data[container_1.ContainerRegistrationGoalDataKey];
        if (registration.input && registration.input.length > 0) {
            try {
                const cp = goalCaching_1.cacheRestore({ entries: registration.input });
                return cp.listener(project, gi, GoalInvocation_1.GoalProjectListenerEvent.before);
            }
            catch (e) {
                const message = `Failed to restore cache input to container for goal ${gi.goal.uniqueName}: ${e.message}`;
                gi.progressLog.write(message);
                return { code: 1, message };
            }
        }
        else {
            return { code: 0, message: "No container input cache entries to restore" };
        }
    },
};
/** Deterministic name for Kubernetes container goal fulfiller. */
exports.K8sContainerFulfillerName = "Kubernetes Container Goal Fulfiller";
/**
 * Goal that fulfills requested container goals by scheduling them as
 * Kubernetes jobs.
 */
function k8sContainerFulfiller() {
    return new GoalWithFulfillment_1.GoalWithFulfillment({
        displayName: exports.K8sContainerFulfillerName,
        uniqueName: exports.K8sContainerFulfillerName,
    })
        .with({
        goalExecutor: containerExecutor,
        name: `${exports.K8sContainerFulfillerName} Executor`,
    })
        .withProjectListener(containerFulfillerCacheRestore);
}
exports.k8sContainerFulfiller = k8sContainerFulfiller;
/**
 * Wait for container in pod to start, return when it does.
 *
 * @param container Information about container to check
 * @param attempts Maximum number of attempts, waiting 500 ms between
 */
async function containerStarted(container, attempts = 240) {
    var _a, _b, _c, _d, _e;
    let core;
    try {
        core = container.config.makeApiClient(k8s.CoreV1Api);
    }
    catch (e) {
        e.message = `Failed to create Kubernetes core API client: ${e.message}`;
        container.log.write(e.message);
        throw e;
    }
    const sleepTime = 500; // ms
    for (let i = 0; i < attempts; i++) {
        await poll_1.sleep(sleepTime);
        let pod;
        try {
            pod = (await core.readNamespacedPod(container.pod, container.ns)).body;
        }
        catch (e) {
            container.log.write(`Reading pod ${container.ns}/${container.pod} failed: ${error_1.k8sErrMsg(e)}`);
            continue;
        }
        const containerStatus = (_b = (_a = pod.status) === null || _a === void 0 ? void 0 : _a.containerStatuses) === null || _b === void 0 ? void 0 : _b.find(c => c.name === container.name);
        if (containerStatus && (!!((_d = (_c = containerStatus.state) === null || _c === void 0 ? void 0 : _c.running) === null || _d === void 0 ? void 0 : _d.startedAt) || !!((_e = containerStatus.state) === null || _e === void 0 ? void 0 : _e.terminated))) {
            const message = `Container '${container.name}' started`;
            container.log.write(message);
            return;
        }
    }
    const errMsg = `Container '${container.name}' failed to start within ${attempts * sleepTime} ms`;
    container.log.write(errMsg);
    throw new Error(errMsg);
}
/**
 * Watch pod until container `container.name` exits and its log stream
 * is done being written to.  Resolve promise with status if container
 * `container.name` exits with status 0.  If container exits with
 * non-zero status, reject promise and includ pod status in the
 * `podStatus` property of the error.  If any other error occurs,
 * e.g., a watch or log error or timeout exceeded, reject immediately
 * upon receipt of error.
 *
 * @param container Information about container to watch
 * @param timeout Milliseconds to allow container to run
 * @return Status of pod after container terminates
 */
function containerWatch(container, timeout) {
    return new Promise(async (resolve, reject) => {
        const clean = {};
        const k8sLog = new k8s.Log(container.config);
        clean.logStream = new stream_1.Writable({
            write: (chunk, encoding, callback) => {
                container.log.write(chunk.toString());
                callback();
            },
        });
        let logDone = false;
        let podStatus;
        let podError;
        const doneCallback = (e) => {
            logDone = true;
            if (e) {
                e.message = `Container logging error: ${error_1.k8sErrMsg(e)}`;
                container.log.write(e.message);
                containerCleanup(clean);
                reject(e);
            }
            if (podStatus) {
                containerCleanup(clean);
                resolve(podStatus);
            }
            else if (podError) {
                containerCleanup(clean);
                reject(podError);
            }
        };
        const logOptions = { follow: true };
        clean.logRequest = await k8sLog.log(container.ns, container.pod, container.name, clean.logStream, doneCallback, logOptions);
        let watch;
        try {
            watch = new k8s.Watch(container.config);
        }
        catch (e) {
            e.message = `Failed to create Kubernetes watch client: ${e.message}`;
            container.log.write(e.message);
            containerCleanup(clean);
            reject(e);
        }
        clean.timeout = setTimeout(() => {
            containerCleanup(clean);
            reject(new Error(`Goal timeout '${timeout}' exceeded`));
        }, timeout);
        const watchPath = `/api/v1/watch/namespaces/${container.ns}/pods/${container.pod}`;
        clean.watcher = await watch.watch(watchPath, {}, async (phase, obj) => {
            var _a, _b;
            const pod = obj;
            if ((_a = pod === null || pod === void 0 ? void 0 : pod.status) === null || _a === void 0 ? void 0 : _a.containerStatuses) {
                const containerStatus = pod.status.containerStatuses.find(c => c.name === container.name);
                if ((_b = containerStatus === null || containerStatus === void 0 ? void 0 : containerStatus.state) === null || _b === void 0 ? void 0 : _b.terminated) {
                    const exitCode = containerStatus.state.terminated.exitCode;
                    if (exitCode === 0) {
                        podStatus = pod.status;
                        const msg = `Container '${container.name}' exited with status 0`;
                        container.log.write(msg);
                        if (logDone) {
                            containerCleanup(clean);
                            resolve(podStatus);
                        }
                    }
                    else {
                        const msg = `Container '${container.name}' exited with status ${exitCode}`;
                        container.log.write(msg);
                        podError = new Error(msg);
                        podError.podStatus = pod.status;
                        if (logDone) {
                            containerCleanup(clean);
                            reject(podError);
                        }
                    }
                    return;
                }
            }
            container.log.write(`Container '${container.name}' phase: ${phase}`);
        }, () => containerCleanup(clean), err => {
            err.message = `Container watcher failed: ${err.message}`;
            container.log.write(err.message);
            containerCleanup(clean);
            reject(err);
        });
    });
}
/** Clean up resources used to watch running container. */
function containerCleanup(c) {
    var _a, _b, _c;
    if (c.timeout) {
        clearTimeout(c.timeout);
    }
    if ((_a = c.logRequest) === null || _a === void 0 ? void 0 : _a.abort) {
        c.logRequest.abort();
    }
    if ((_b = c.logStream) === null || _b === void 0 ? void 0 : _b.end) {
        c.logStream.end();
    }
    if ((_c = c.watcher) === null || _c === void 0 ? void 0 : _c.abort) {
        c.watcher.abort();
    }
}
//# sourceMappingURL=container.js.map