Jenkinsfile

pipeline {
    agent {
        label 'PerfCi'
    }
    environment {
        UPGRADE_OCP_VERSION = credentials('perfci_upgrade_target_ocp_version')
        CNV_VERSION = credentials('perfci_upgrade_target_cnv_version')
        LSO_VERSION = credentials('perfci_upgrade_target_lso_version')
        ODF_VERSION = credentials('perfci_upgrade_target_odf_version')
        PROVISION_USER = credentials('perfci_provision_user')
        PROVISION_IP = credentials('perfci_provision_ip')
        WAIT_FOR_UPGRADE_VERSION = credentials('perfci_upgrade_target_ocp_version')
        UPGRADE_CHANNEL = credentials('perfci_upgrade_channel') // stable (default)/candidate
        CONTACT_EMAIL1 = credentials('perfci_contact_email1')
        CLUSTER_TYPE = credentials('perfci_cluster_type')
        EXPECTED_NODES = credentials('perfci_expected_nodes')
        OCP_CLIENT_VERSION = credentials('perfci_ocp_client_version')
        QUAY_BENCHMARK_RUNNER_REPOSITORY = credentials('perfci_quay_repository')
        WORKSPACE = '/home/jenkins'
        KUBEADMIN_PASSWORD_PATH = '/home/jenkins/.kube/kubeadmin-password'
        KUBECONFIG_PATH = '/root/.kube/config'
        PRIVATE_KEY_PATH = '/home/jenkins/.ssh/provision_private_key'
        CONFIG_PATH = '/home/jenkins/.ssh/config'
        CONTAINER_CONFIG_PATH = '/root/.ssh/config'
        ANSIBLE_TMP_PATH = '/root/.ansible/tmp/'
        SCALE_NODES = credentials('perfci_scale_nodes')
        PIN_NODE_BENCHMARK_OPERATOR = credentials('perfci_pin_node_benchmark_operator')
        PIN_NODE1 = credentials('perfci_pin_node1')
        PIN_NODE2 = credentials('perfci_pin_node2')
        ELASTICSEARCH = credentials('perfci_elasticsearch')
        ELASTICSEARCH_PORT = credentials('perfci_elasticsearch_port')
        IBM_REGION_NAME = credentials('perfci_ibm_region_name')
        IBM_ENDPOINT_URL = credentials('perfci_ibm_endpoint_url')
        IBM_ACCESS_KEY_ID = credentials('perfci_ibm_access_key_id')
        IBM_SECRET_ACCESS_KEY = credentials('perfci_ibm_secret_access_key')
        IBM_BUCKET = credentials('perfci_ibm_bucket')
        IBM_KEY = credentials('perfci_ibm_key')
        GOOGLE_DRIVE_PATH = credentials('perfci_google_drive_path')
        GOOGLE_DRIVE_CREDENTIALS_FILE = credentials('perfci_google_drive_credentials_file')
        GOOGLE_DRIVE_TOKEN_FILE = credentials('perfci_google_drive_token_file')
        GOOGLE_CREDENTIALS_FILE = 'credentials.json'
        GOOGLE_TOKEN_FILE = 'token.json'
        GOOGLE_DESTINATION_PATH = '/tmp'
        GOOGLE_DRIVE_SHARED_DRIVE_ID = credentials('perfci_google_drive_shared_drive_id')
        RUN_ARTIFACTS_URL = credentials('perfci_run_artifacts_url')
        REDIS = credentials('perfci_redis')
        WORKER_DISK_IDS = credentials('perfci_worker_disk_ids')
        WINDOWS10_URL = credentials('perfci_windows10_url')
        WINDOWS11_URL = credentials('perfci_windows11_url')
        WINDOWS_SERVER_2019_URL = credentials('perfci_windows_server_2019_url')
        WINDOWS_SERVER_2022_URL = credentials('perfci_windows_server_2022_url')
        LSO_NODE = credentials('perfci_lso_node')
        KUBEADMIN_PASSWORD = readFile('/home/jenkins/.kube/kubeadmin-password').trim()
        WORKER_DISK_PREFIX = 'wwn-0x'
        SAVE_ARTIFACTS_LOCAL = 'False'
        ENABLE_PROMETHEUS_SNAPSHOT = 'False'
        DELETE_ALL = 'False' // Not delete the running Windows11 VMs
        VERIFICATION_ONLY = 'True'
        RUN_TYPE = 'perf_ci'
        NUM_ODF_DISK = 6
        WINDOWS_SCALE = 40
        THREADS_LIMIT = 20
        PROVISION_TIMEOUT = 7200
        TIMEOUT = 7200
        WORKSPACE_PATH = "${env.WORKSPACE}/workspace/${env.JOB_NAME}/"
    }

    stages {
        stage('Cleanup') {
            steps {
                script {
                    try {
                        // Clean the Jenkins workspace
                        echo "Cleaning Jenkins workspace"
                        deleteDir()

                    } catch (Exception e) {
                        echo "Error: ${e.getMessage()}"
                        // Additional error handling or actions can be added here
                    }
                }
            }
        }

        stage('⚙️ SET SSH key') {
            steps {
                script {
                    sh 'mkdir -p $WORKSPACE/.ssh/'
                    sh 'sudo chmod 700 $WORKSPACE/.ssh/'
                    withCredentials([file(credentialsId: 'perfci_provision_private_key_file', variable: 'PROVISION_PRIVATE_KEY_FILE')]) {
                        sh "sudo cp ${PROVISION_PRIVATE_KEY_FILE} ${WORKSPACE}/.ssh/provision_private_key"
                        sh "sudo chown jenkins:jenkins ${WORKSPACE}/.ssh/provision_private_key"
                    }
                    sh 'sudo chmod 600 $PRIVATE_KEY_PATH'
                    sh '''
                        sudo cat > "$CONFIG_PATH" <<END
    Host provision
        HostName ${PROVISION_IP}
        User ${PROVISION_USER}
        IdentityFile $PRIVATE_KEY_PATH
        StrictHostKeyChecking no
        ServerAliveInterval 30
        ServerAliveCountMax 5
END
                    '''
                    sh 'sudo chmod 600 $CONFIG_PATH'
                }
            }
        }

        stage('Prepare Google Credential Files') {
            steps {
                script {
                    // Create a unique temp directory
                    env.TEMP_DIR = sh(script: "umask 77; mktemp -d -p '${env.WORKSPACE_PATH}'", returnStdout: true).trim()
                    if (!env.TEMP_DIR) {
                        error "Failed to create temporary directory!"
                    }

                    // Set file paths
                    env.GOOGLE_DRIVE_CREDENTIALS = "${env.TEMP_DIR}/${GOOGLE_CREDENTIALS_FILE}"
                    env.GOOGLE_DRIVE_TOKEN = "${env.TEMP_DIR}/${GOOGLE_TOKEN_FILE}"

                    // Prepare credential files
                    sh """
                        set -e
                        umask 77
                        cp '${GOOGLE_DRIVE_CREDENTIALS_FILE}' '${env.GOOGLE_DRIVE_CREDENTIALS}' || error "Failed to copy ${GOOGLE_DRIVE_CREDENTIALS_FILE}"
                        cp '${GOOGLE_DRIVE_TOKEN_FILE}' '${env.GOOGLE_DRIVE_TOKEN}' || error "Failed to copy ${GOOGLE_DRIVE_TOKEN_FILE}"
                    """
                }
            }
        }

        stage('OpenShift Upgrade AND Verifications') {
            parallel {
                stage('Upgrade Steps') {
                    steps {
                        script {
                            def upgrade_steps = ['run_bare_metal_ocp_upgrade', 'verify_bare_metal_upgrade_complete']

                            for (upgrade_step in upgrade_steps) {
                                echo "Run OpenShift Deployment for ${upgrade_step}"

                                // Create a stage for each resource
                                stage("${upgrade_step} Deployment") {
                                    echo "Running deployment for ${upgrade_step}"
                                    // deployment code
                                    script {
                                                    sh """
                                                        sudo podman run --rm \
                                                            -e UPGRADE_OCP_VERSION="${UPGRADE_OCP_VERSION}" \
                                                            -e UPGRADE_STEP="${upgrade_step}" \
                                                            -e UPGRADE_CHANNEL='${UPGRADE_CHANNEL}' \
                                                            -e LSO_VERSION="${LSO_VERSION}" \
                                                            -e ODF_VERSION="${ODF_VERSION}" \
                                                            -e CNV_VERSION="${CNV_VERSION}" \
                                                            -e EXPECTED_NODES='${EXPECTED_NODES}' \
                                                            -e KUBEADMIN_PASSWORD="${KUBEADMIN_PASSWORD}" \
                                                            -e KUBECONFIG_PATH="${KUBECONFIG_PATH}" \
                                                            -e TIMEOUT="${TIMEOUT}" \
                                                            -v "${KUBECONFIG_PATH}":"${KUBECONFIG_PATH}" \
                                                            --privileged "${QUAY_BENCHMARK_RUNNER_REPOSITORY}"

                                                    """
                                    }
                                }
                            }
                        }
                    }
                }

            // Parallel stage
            stage('WORKLOADS Verifications') {
            steps {
                script {
                    def workloads =  [ 'windows_vm_scale_windows11' ]
                    def build_version = sh(script: 'curl -s "https://pypi.org/pypi/benchmark-runner/json" | jq -r .info.version', returnStdout: true).trim()
                    def WINDOWS_URL = ''
                    def SCALE = ''
                    def workload_name = ''
                    for (workload in workloads) {
                            boolean success = false
                            try {
                                // workload full name
                                workload_name = workload
                                // Parse the workload name
                                def parts = "${workload}".split("_")
                                def WORKLOAD = parts[0] + "_" + parts[1]
                                def RUN = parts.size() >= 3 ? parts[2] : ""
                                if (RUN == "scale") {

                                    if (WORKLOAD == "windows_vm") {
                                        switch (workload) {
                                            case "windows_vm_scale_windows10":
                                                WINDOWS_URL = WINDOWS10_URL
                                                break
                                            case "windows_vm_scale_windows11":
                                                WINDOWS_URL = WINDOWS11_URL
                                                break
                                            case "windows_vm_scale_windows_server_2019":
                                                WINDOWS_URL = WINDOWS_SERVER_2019_URL
                                                break
                                            case "windows_vm_scale_windows_server_2022":
                                                WINDOWS_URL = WINDOWS_SERVER_2022_URL
                                                break
                                            default:
                                                error "Unknown Windows scale workload ${workload}"
                                        }
                                        SCALE = WINDOWS_SCALE
                                    }
                                  workload = WORKLOAD
                                } else {
                                    SCALE = ""
                                }

                                // Windows11 VMs workload Verifications
                                stage(workload_name) {
                                    // boolean value w/o ''
                                    sh """
                                        sudo podman run --rm -t \
                                        -e WORKLOAD='${workload}' \
                                        -e ODF_VERSION="${ODF_VERSION}" \
                                        -e CNV_VERSION="${CNV_VERSION}" \
                                        -e WAIT_FOR_UPGRADE_VERSION='${WAIT_FOR_UPGRADE_VERSION}' \
                                        -e KUBEADMIN_PASSWORD='${KUBEADMIN_PASSWORD}' \
                                        -e KUBECONFIG_PATH='${KUBECONFIG_PATH}' \
                                        -e PIN_NODE_BENCHMARK_OPERATOR='${PIN_NODE_BENCHMARK_OPERATOR}' \
                                        -e PIN_NODE1='${PIN_NODE1}' \
                                        -e PIN_NODE2='${PIN_NODE2}' \
                                        -e ELASTICSEARCH='${ELASTICSEARCH}' \
                                        -e ELASTICSEARCH_PORT='${ELASTICSEARCH_PORT}' \
                                        -e IBM_REGION_NAME='${IBM_REGION_NAME}' \
                                        -e IBM_ENDPOINT_URL='${IBM_ENDPOINT_URL}' \
                                        -e IBM_ACCESS_KEY_ID='${IBM_ACCESS_KEY_ID}' \
                                        -e IBM_SECRET_ACCESS_KEY='${IBM_SECRET_ACCESS_KEY}' \
                                        -e IBM_BUCKET='${IBM_BUCKET}' \
                                        -e IBM_KEY='${IBM_KEY}' \
                                        -e GOOGLE_DRIVE_PATH='${GOOGLE_DRIVE_PATH}' \
                                        -e GOOGLE_DRIVE_CREDENTIALS_FILE='${GOOGLE_DESTINATION_PATH}/${GOOGLE_CREDENTIALS_FILE}' \
                                        -e GOOGLE_DRIVE_TOKEN_FILE='${GOOGLE_DESTINATION_PATH}/${GOOGLE_TOKEN_FILE}' \
                                        -e GOOGLE_DRIVE_SHARED_DRIVE_ID='${GOOGLE_DRIVE_SHARED_DRIVE_ID}' \
                                        -e RUN_ARTIFACTS_URL='${RUN_ARTIFACTS_URL}' \
                                        -e BUILD_VERSION='${build_version}' \
                                        -e RUN_TYPE='${RUN_TYPE}' \
                                        -e SAVE_ARTIFACTS_LOCAL=${SAVE_ARTIFACTS_LOCAL} \
                                        -e ENABLE_PROMETHEUS_SNAPSHOT=${ENABLE_PROMETHEUS_SNAPSHOT} \
                                        -e WORKER_DISK_IDS='${WORKER_DISK_IDS}' \
                                        -e WORKER_DISK_PREFIX='${WORKER_DISK_PREFIX}' \
                                        -e SCALE='${SCALE}' \
                                        -e SCALE_NODES='${SCALE_NODES}' \
                                        -e REDIS='${REDIS}' \
                                        -e THREADS_LIMIT='${THREADS_LIMIT}' \
                                        -e WINDOWS_URL='${WINDOWS_URL}'  \
                                        -e DELETE_ALL=${DELETE_ALL} \
                                        -e VERIFICATION_ONLY=${VERIFICATION_ONLY} \
                                        -e LSO_NODE='${LSO_NODE}'  \
                                        -e TIMEOUT='${TIMEOUT}' \
                                        -e log_level='INFO' \
                                        -v '${KUBECONFIG_PATH}:${KUBECONFIG_PATH}' \
                                        -v '${TEMP_DIR}:${GOOGLE_DESTINATION_PATH}' \
                                        --privileged '${QUAY_BENCHMARK_RUNNER_REPOSITORY}'
                                    """
                                    success = true
                                }
                            } catch (Exception e) {
                                echo "Error occurred in workload ${workload}: ${e.message}"
                                if (i < retryCount - 1) {
                                    echo "Retrying..."
                                    sleep(time: 120, unit: 'SECONDS') // Sleep for 2 minutes between retries
                                } else {
                                    echo "Reached maximum retry attempts for workload ${workload}. Moving to the next workload..."
                                }
                        }
                        if (!success) {
                            echo "Workload ${workload} failed after ${retryCount} retries."
                        }
                        // sleep 10 sec between each VMs validation interation
                        sleep(time: 10, unit: 'SECONDS')
                 }
                }
            }
        } // WORKLOADS Verifications
    }
        }
    } // stages

    post {
        always {
            script {
                sh '''
                    # Check if the image exists
                    if [[ "$(sudo podman images -q ${QUAY_BENCHMARK_RUNNER_REPOSITORY} 2> /dev/null)" != "" ]]; then

                        # Get containers using the image
                        containers=$(sudo podman ps -a --filter ancestor=${QUAY_BENCHMARK_RUNNER_REPOSITORY} -q)

                        if [[ "$containers" != "" ]]; then
                            echo "Stopping and removing containers using the image..."
                            sudo podman stop $containers
                            sudo podman rm -f $containers
                        fi

                        # Now remove the image
                        sudo podman rmi -f $(sudo podman images -q ${QUAY_BENCHMARK_RUNNER_REPOSITORY} 2> /dev/null)
                    fi
                    # Clean up the virtual environment
                    rm -rf venv
                    # Clean temp dir
                    if [ -d "${TEMP_DIR}" ]; then
                        sudo rm -rf "${TEMP_DIR}"
                        echo "Temporary directory '${TEMP_DIR}' deleted."
                    fi
                '''
            }
        }
        failure {
            script {
                msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})"
                emailext body: """\
                    Jenkins job: ${env.BUILD_URL}\nSee the console output for more details:  ${env.BUILD_URL}consoleFull\n\n
                """, subject: msg, to: "${CONTACT_EMAIL1}"
            }
        }
        success {
            echo 'Triggering 05-PerfCI-Chaos-Verify-Windows-VMs-Deployment pipeline'
            build job: '05-PerfCI-Chaos-Verify-Windows-VMs-Deployment', wait: false
        }
    }
}