diff --git a/vars/common.groovy b/vars/common.groovy index c36b0d519..0312d9490 100644 --- a/vars/common.groovy +++ b/vars/common.groovy @@ -158,30 +158,22 @@ def get_branch_information() { for (platform in linux_platforms) { get_docker_image(platform) - def all_sh_help = sh( + available_all_sh_components[platform] = sh( script: docker_script( - platform, "./tests/scripts/all.sh", "--help" + platform, "./tests/scripts/all.sh", "--list-components" ), + label: "./tests/scripts/all.sh --list-components #$platform", returnStdout: true - ) - if (all_sh_help.contains('list-components')) { - available_all_sh_components[platform] = sh( + ).trim().split('\n') + if (all_all_sh_components == []) { + all_all_sh_components = sh( script: docker_script( - platform, "./tests/scripts/all.sh", "--list-components" + platform, "./tests/scripts/all.sh", + "--list-all-components" ), + label: "./tests/scripts/all.sh --list-all-components", returnStdout: true ).trim().split('\n') - if (all_all_sh_components == []) { - all_all_sh_components = sh( - script: docker_script( - platform, "./tests/scripts/all.sh", - "--list-all-components" - ), - returnStdout: true - ).trim().split('\n') - } - } else { - error('Pre Test Checks failed: Base branch out of date. Please rebase') } } } @@ -245,7 +237,7 @@ def maybe_notify_github(context, state, description) { } def archive_zipped_log_files(job_name) { - sh """\ + sh label: "rename+compress *.log", script: """\ for i in *.log; do [ -f "\$i" ] || break mv "\$i" "$job_name-\$i" diff --git a/vars/gen_jobs.groovy b/vars/gen_jobs.groovy index 9b71c084b..1f6c1ca3f 100644 --- a/vars/gen_jobs.groovy +++ b/vars/gen_jobs.groovy @@ -36,7 +36,7 @@ def gen_simple_windows_jobs(label, script) { checkout_repo.checkout_repo() timeout(time: common.perJobTimeout.time, unit: common.perJobTimeout.unit) { - bat script + bat script: script, label: label } } } catch (err) { @@ -50,86 +50,6 @@ def gen_simple_windows_jobs(label, script) { return jobs } -def gen_docker_jobs_foreach(label, platforms, compilers, script) { - def jobs = [:] - - for (platform in platforms) { - for (compiler in compilers) { - def job_name = "${label}-${compiler}-${platform}" - def shell_script = sprintf(script, common.compiler_paths[compiler]) - jobs[job_name] = { - node('container-host') { - try { - deleteDir() - common.get_docker_image(platform) - dir('src') { - checkout_repo.checkout_repo() - writeFile file: 'steps.sh', text: """\ -#!/bin/sh -set -eux -ulimit -f 20971520 -${shell_script} -""" - sh 'chmod +x steps.sh' - } - timeout(time: common.perJobTimeout.time, - unit: common.perJobTimeout.unit) { - try { - sh common.docker_script( - platform, "/var/lib/build/steps.sh" - ) - } finally { - dir('src/tests/') { - common.archive_zipped_log_files(job_name) - } - } - } - } catch (err) { - failed_builds[job_name] = true - throw (err) - } finally { - deleteDir() - } - } - } - } - } - return jobs -} - -def gen_node_jobs_foreach(label, platforms, compilers, script) { - def jobs = [:] - - for (platform in platforms) { - for (compiler in compilers) { - def job_name = "${label}-${compiler}-${platform}" - def shell_script = sprintf(script, common.compiler_paths[compiler]) - jobs[job_name] = { - node(platform) { - try { - deleteDir() - checkout_repo.checkout_repo() - shell_script = """\ -ulimit -f 20971520 -export PYTHON=/usr/local/bin/python2.7 -""" + shell_script - timeout(time: common.perJobTimeout.time, - unit: common.perJobTimeout.unit) { - sh shell_script - } - } catch (err) { - failed_builds[job_name] = true - throw (err) - } finally { - deleteDir() - } - } - } - } - } - return jobs -} - def node_label_for_platform(platform) { switch (platform) { case ~/^(debian|ubuntu)(-.*)?/: return 'container-host'; @@ -234,12 +154,13 @@ ${extra_setup_code} unit: common.perJobTimeout.unit) { try { if (use_docker) { - sh common.docker_script( + sh script: common.docker_script( platform, "/var/lib/build/steps.sh" - ) + ), label: "docker run steps.sh #${job_name}" } else { dir('src') { - sh './steps.sh' + sh script: './steps.sh', + label: "docker run steps.sh #${job_name}" } } } finally { @@ -345,7 +266,8 @@ def gen_abi_api_checking_job(platform) { /* The credentials here are the SSH credentials for accessing the repositories. They are defined at {JENKINS_URL}/credentials */ withCredentials([sshUserPrivateKey(credentialsId: credentials_id, keyFileVariable: 'keyfile')]) { - sh "GIT_SSH_COMMAND=\"ssh -i ${keyfile}\" git fetch origin ${CHANGE_TARGET}" + sh script: "GIT_SSH_COMMAND=\"ssh -i ${keyfile}\" git fetch origin ${CHANGE_TARGET}", + label: "git fetch origin ${CHANGE_TARGET}" } writeFile file: 'steps.sh', text: """\ #!/bin/sh @@ -363,9 +285,9 @@ scripts/abi_check.py -o FETCH_HEAD -n HEAD -s identifiers --brief } timeout(time: common.perJobTimeout.time, unit: common.perJobTimeout.unit) { - sh common.docker_script( + sh script: common.docker_script( platform, "/var/lib/build/steps.sh" - ) + ), label: "steps.sh #${job_name}" } } catch (err) { failed_builds[job_name] = true @@ -414,9 +336,9 @@ fi timeout(time: common.perJobTimeout.time, unit: common.perJobTimeout.unit) { try { - sh common.docker_script( + sh script: common.docker_script( platform, "/var/lib/build/steps.sh" - ) + ), label: "steps.sh #${job_name}" dir('src') { String coverage_log = readFile('coverage-summary.txt') coverage_details['coverage'] = coverage_log.substring( @@ -472,7 +394,7 @@ def gen_mbed_os_example_job(repo, branch, example, compiler, platform, raas) { try { deleteDir() /* Create python virtual environment and install mbed tools */ - sh """\ + sh label: "virtualenv #${job_name}", script: """\ ulimit -f 20971520 virtualenv $WORKSPACE/mbed-venv . $WORKSPACE/mbed-venv/bin/activate @@ -486,7 +408,7 @@ pip install mbed-host-tests /* If the job is targeting an example repo, then we wish to use the versions * of Mbed OS, TLS and Crypto specified by the mbed-os.lib file. */ if (env.TARGET_REPO == 'example') { - sh """\ + sh label: "mbed deploy #${job_name}", script: """\ ulimit -f 20971520 . $WORKSPACE/mbed-venv/bin/activate mbed config root . @@ -498,7 +420,7 @@ mbed deploy -vv * checking it out twice. Mbed deploy is still run in case other libraries * are required to be deployed. We then check out Mbed OS, TLS and Crypto * according to the job parameters. */ - sh """\ + sh label: "rm mbed-os.lib; mbed deploy #${job_name}", script: """\ ulimit -f 20971520 . $WORKSPACE/mbed-venv/bin/activate rm -f mbed-os.lib @@ -509,7 +431,7 @@ mbed deploy -vv deleteDir() checkout_repo.checkout_mbed_os() /* Check that python requirements are up to date */ - sh """\ + sh label: "pip install #${job_name}", script: """\ ulimit -f 20971520 . $WORKSPACE/mbed-venv/bin/activate pip install -r requirements.txt @@ -523,14 +445,14 @@ pip install -r requirements.txt if (example == 'atecc608a') { tag_filter = "--tag-filters HAS_CRYPTOKIT" } - sh """\ + sh label: "mbed compile #${job_name}", script: """\ ulimit -f 20971520 . $WORKSPACE/mbed-venv/bin/activate mbed compile -m ${platform} -t ${compiler} """ for (int attempt = 1; attempt <= 3; attempt++) { try { - sh """\ + sh label: "mbedhtrun #${job_name}#${attempt}", script: """\ ulimit -f 20971520 if [ -e BUILD/${platform}/${compiler}/${example}.bin ] then @@ -653,7 +575,9 @@ def gen_dockerfile_builder_job(platform, overwrite=false) { node('dockerfile-builder') { def image_exists = false if (!overwrite) { - image_exists = sh(script: check_docker_image, returnStatus: true) == 0 + image_exists = sh(script: check_docker_image, + label: "check_docker_image ${tag}", + returnStatus: true) == 0 } if (overwrite || !image_exists) { dir('docker') { @@ -665,7 +589,7 @@ def gen_dockerfile_builder_job(platform, overwrite=false) { extra_build_args = '--build-arg ARMLMD_LICENSE_FILE=27000@flexnet.trustedfirmware.org' withCredentials([string(credentialsId: 'DOCKER_AUTH', variable: 'TOKEN')]) { - sh """\ + sh label: ">.docker/config.json #${platform}", script: """\ mkdir -p ${env.HOME}/.docker cat > ${env.HOME}/.docker/config.json << EOF { @@ -680,12 +604,12 @@ chmod 0600 ${env.HOME}/.docker/config.json """ } } else { - sh """\ + sh label: "docker login #${platform}", script: """\ aws ecr get-login-password | docker login --username AWS --password-stdin $common.docker_ecr """ } - sh """\ + sh label: "docker push #${tag}", script: """\ # Use BuildKit and a remote build cache to pull only the reuseable layers # from the last successful build for this platform DOCKER_BUILDKIT=1 docker build \