// "prod-jenkins-aws-task-slave01" node is specified in order to have access
// to both Prod and QA memcached nodes.
node('aws') {
    // Prepare build params for chosen environment
    def params_list = params["${ENVIRONMENT}_PARAMS"].split("\n")
    def build_params = [:]
    for (p in params_list ) {
        def (key, value) = p.split("=")
        build_params[key] = value
    }

    def current_active_ddb_read_cap = null
    def current_active_ddb_write_cap = null

    def current_audit_ddb_read_cap = null
    def current_audit_ddb_write_cap = null

    build_params.DDB_READ_CAPACITY=build_params.DDB_READ_CAPACITY as Integer

    def emr_cluster_id = null
    def today = new Date().format("yyyy-MM-dd")
    def s3_path = "${build_params.S3_DIRECTORY}/${today}_${BUILD_ID}"

    def target_active_s3_location = "s3://${build_params.S3_BUCKET}/${s3_path}/data/active/"
    def ddb_active_transform_script_location = "s3://${build_params.S3_BUCKET}/${s3_path}/${build_params.DDB_ACTIVE_TRANSFORM_SCRIPT_NAME}"

    def target_audit_s3_location = "s3://${build_params.S3_BUCKET}/${s3_path}/data/audit/"
    def ddb_audit_transform_script_location = "s3://${build_params.S3_BUCKET}/${s3_path}/${build_params.DDB_AUDIT_TRANSFORM_SCRIPT_NAME}"

    stage('Prepare') {
        checkout scm
        echo 'Substitute parameters in hive scripts'
        echo 'MR_ACTIVE table'
        sh "cp hive/${build_params.HIVE_SCRIPT} hive/${build_params.ACTIVE_HIVE_SCRIPT}.shadow"
        sh """sed -e "s|\\[source_ddb_table_name]|${build_params.SOURCE_ACTIVE_DDB_TABLE_NAME}|" \\
         -e "s|\\[target_s3_location]|${target_active_s3_location}|" \\
         -e "s|\\[ddb_transform_script_location]|${ddb_active_transform_script_location}|" \\
         -e "s|\\[ddb_transform_script_name]|${build_params.DDB_ACTIVE_TRANSFORM_SCRIPT_NAME}|" \\
         -e "s|\\[s3_hive_table]|s3_mr_active|" \\
         -e "s|\\[ddb_hive_table]|ddb_mr_active|" \\
          hive/${build_params.ACTIVE_HIVE_SCRIPT}.shadow > hive/${build_params.ACTIVE_HIVE_SCRIPT}"""
        sh "cat hive/${build_params.ACTIVE_HIVE_SCRIPT}"

        echo 'MR_AUDIT table'
        sh "cp hive/${build_params.HIVE_SCRIPT} hive/${build_params.AUDIT_HIVE_SCRIPT}.shadow"
        sh """sed -e "s|\\[source_ddb_table_name]|${build_params.SOURCE_AUDIT_DDB_TABLE_NAME}|" \\
         -e "s|\\[target_s3_location]|${target_audit_s3_location}|" \\
         -e "s|\\[ddb_transform_script_location]|${ddb_audit_transform_script_location}|" \\
         -e "s|\\[ddb_transform_script_name]|${build_params.DDB_AUDIT_TRANSFORM_SCRIPT_NAME}|" \\
         -e "s|\\[s3_hive_table]|s3_mr_audit|" \\
         -e "s|\\[ddb_hive_table]|ddb_mr_audit|" \\
          hive/${build_params.AUDIT_HIVE_SCRIPT}.shadow > hive/${build_params.AUDIT_HIVE_SCRIPT}"""
        sh "cat hive/${build_params.AUDIT_HIVE_SCRIPT}"
    }
    stage('S3 Upload') {
        echo 'Upload hive and transform script to S3'
        withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: build_params.AWS_ACCESS_KEY_ID]]) {
            sh "aws s3 cp hive/${build_params.ACTIVE_HIVE_SCRIPT} s3://${build_params.S3_BUCKET}/${s3_path}/"
            sh "aws s3 cp dynamodb_parse/${build_params.DDB_ACTIVE_TRANSFORM_SCRIPT_NAME} s3://${build_params.S3_BUCKET}/${s3_path}/"
            echo 'Scripts for MR_ACTIVE uploaded'
            sh "aws s3 cp hive/${build_params.AUDIT_HIVE_SCRIPT} s3://${build_params.S3_BUCKET}/${s3_path}/"
            sh "aws s3 cp dynamodb_parse/${build_params.DDB_AUDIT_TRANSFORM_SCRIPT_NAME} s3://${build_params.S3_BUCKET}/${s3_path}/"
            echo 'Scripts for MR_AUDIT uploaded'
        }
    }
    try {
        stage('Toggle MR Maintenance') {
            echo 'Toggle MasterRegistry Maintenance state'
            sh 'python3 -m venv maintenance_env'
            sh 'maintenance_env/bin/pip install -r maintenance/requirements.txt'
            sh "maintenance_env/bin/python maintenance/toggle_maintenance.py start -h ${build_params.MEMCACHED_HOST} -p ${build_params.MEMCACHED_PORT}"
        }
        stage('Launch EMR Cluster') {
            withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: build_params.AWS_ACCESS_KEY_ID]]) {
                emr_cluster_id = sh (
                    returnStdout: true,
                    script: """
                        aws emr create-cluster --applications Name=Hadoop Name=Hive Name=Tez --ebs-root-volume-size 10 \
                          --ec2-attributes '{ \
                            \"KeyName\":\"${build_params.EMR_KEY_NAME}\", \
                            \"InstanceProfile\":\"${build_params.EMR_INSTANCE_PROFILE}\", \
                            \"ServiceAccessSecurityGroup\":\"${build_params.EMR_SERVICE_ACCESS_SECURITY_GROUP}\", \
                            \"SubnetId\":\"${build_params.EMR_SUBNET_ID}\", \
                            \"EmrManagedSlaveSecurityGroup\":\"${build_params.EMR_MANAGED_SLAVE_SECURITY_GROUP}\", \
                            \"EmrManagedMasterSecurityGroup\":\"${build_params.EMR_MANAGED_MASTER_SECURITY_GROUP}\"  }' \
                          --service-role ${build_params.EMR_SERVICE_ROLE} \
                          --log-uri s3://${build_params.S3_BUCKET}/${s3_path}/logs/ \
                          --release-label emr-5.25.0 \
                          --steps '[ \
                              {\"Args\":[\"hive-script\", \"--run-hive-script\", \"--args\", \"-f\", \"s3://${build_params.S3_BUCKET}/${s3_path}/${build_params.ACTIVE_HIVE_SCRIPT}\"], \"Type\":\"CUSTOM_JAR\", \"ActionOnFailure\":\"TERMINATE_CLUSTER\", \"Jar\":\"command-runner.jar\",  \"Properties\":\"\", \"Name\":\"Hive program\"}, \
                              {\"Args\":[\"hive-script\", \"--run-hive-script\", \"--args\", \"-f\", \"s3://${build_params.S3_BUCKET}/${s3_path}/${build_params.AUDIT_HIVE_SCRIPT}\"], \"Type\":\"CUSTOM_JAR\", \"ActionOnFailure\":\"TERMINATE_CLUSTER\", \"Jar\":\"command-runner.jar\",  \"Properties\":\"\", \"Name\":\"Hive program\"} \
                              ]' \
                          --name '${build_params.EMR_CLUSTER_NAME}' \
                          --instance-groups  '[{\"InstanceCount\":${build_params.EMR_CORE_INSTANCE_COUNT}, \"EbsConfiguration\":{ \"EbsBlockDeviceConfigs\":[{\"VolumeSpecification\":{ \"SizeInGB\":32, \"VolumeType\":\"gp2\" }, \"VolumesPerInstance\":1 }]},  \"InstanceGroupType\":\"CORE\", \"InstanceType\":\"${build_params.EMR_CORE_INSTANCE_TYPE}\", \"Name\":\"Core Instance Group\" }, \
                          { \"InstanceCount\":${build_params.EMR_MASTER_INSTANCE_COUNT}, \"EbsConfiguration\":{ \"EbsBlockDeviceConfigs\":[ {\"VolumeSpecification\":{ \"SizeInGB\":32, \"VolumeType\":\"gp2\" }, \"VolumesPerInstance\":1 }]},  \"InstanceGroupType\":\"MASTER\", \"InstanceType\":\"${build_params.EMR_MASTER_INSTANCE_TYPE}\", \"Name\":\"Master Instance Group\"}]' \
                          --scale-down-behavior TERMINATE_AT_TASK_COMPLETION  --auto-terminate  --region us-east-1 --output text --query 'ClusterId'
                        """).trim()
                echo "Cluster started. ID: ${emr_cluster_id}"
            }
        }
        stage('Wait for EMR Cluster Finish') {
            def emr_status = 'STARTING'
            def terminated_status = 'TERMINATED'
            def terminated_with_errors_status = 'TERMINATED_WITH_ERRORS'
            def terminated = [terminated_status, terminated_with_errors_status]

            withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: build_params.AWS_ACCESS_KEY_ID]]) {
                waitUntil() {
                    emr_status = sh (returnStdout: true, script: "aws emr describe-cluster --cluster-id ${emr_cluster_id} --output text --query 'Cluster.Status.State'").trim()
                    echo "Current EMR cluster status: ${emr_status}"
                    return (emr_status in terminated)
                }
                echo "EMR cluster terminated with status: ${emr_status}"
            }
            if (emr_status == terminated_with_errors_status) {
                error('EMR cluster terminated with errors. Export process failed.')
            }
        }
        stage('Launch Snowflake Import') {
            build job: "${SNOWFLAKE_IMPORT_JOB}", parameters: [
                string(name: 'ENVIRONMENT', value: "${ENVIRONMENT}"),
                string(name: 'UPSTREAM_BUILD_ID', value: "${BUILD_ID}")
            ], wait: true
        }
    }
    catch(err) {
        print err
        currentBuild.result = 'FAILURE'
    }
    finally {
        stage('Toggle MR Maintenance') {
            sh "maintenance_env/bin/python maintenance/toggle_maintenance.py stop -h ${build_params.MEMCACHED_HOST} -p ${build_params.MEMCACHED_PORT}"
        }
        stage('Cleanup') {
            deleteDir()
        }
    }
}