diff --git a/developer_guide.md b/developer_guide.md index 37acd137b..1cdf6ba6b 100644 --- a/developer_guide.md +++ b/developer_guide.md @@ -130,7 +130,7 @@ Note that Groovy does not have global variables as such. Each module (`*.groovy` The entry point of the pipeline runs on the Jenkins master node. Because all jobs start on this node, we should not do much on the master node. In particular, we don't check out the code to test on the master. All computation-heavy or I/O-heavy processing must be performed on an executor: ``` -node (label) { +common.mbedtls_node (label) { // IO-heavy or computation-heavy code } ``` diff --git a/vars/analysis.groovy b/vars/analysis.groovy index 66829dc7a..0f47ba860 100644 --- a/vars/analysis.groovy +++ b/vars/analysis.groovy @@ -64,7 +64,7 @@ void record_timestamps(String group, String job_name, Callable body, Strin body() } if (node_label != null) { - node(node_label, stamped_body) + common.mbedtls_node(node_label, stamped_body) } else { stamped_body() } diff --git a/vars/common.groovy b/vars/common.groovy index c0fd4516b..9a96ff0c4 100644 --- a/vars/common.groovy +++ b/vars/common.groovy @@ -39,8 +39,13 @@ import org.kohsuke.github.GHPermissionType import org.mbed.tls.jenkins.BranchInfo -/* Indicates if CI is running on Open CI (hosted on https://ci.trustedfirmware.org/) */ -@Field is_open_ci_env = env.JENKINS_URL ==~ /\S+(trustedfirmware)\S+/ +/* Indicates if CI is running on Open CI (hosted on https://mbedtls.trustedfirmware.org/) */ +@Field final boolean is_open_ci_env = env.JENKINS_URL ==~ /\S+(mbedtls\.trustedfirmware\.org)\S+/ + +/* Indicates if CI is running on the new CI (hosted on https://ci.trustedfirmware.org/) */ +@Field final boolean is_new_ci_env = !is_open_ci_env && (env.JENKINS_URL ==~ /\S+(trustedfirmware)\S+/) + +@Field final String ci_name = is_open_ci_env ? 'TF OpenCI' : is_new_ci_env ? 'New CI (testing)' : 'Internal CI' /* * This controls the timeout each job has. It does not count the time spent in @@ -63,9 +68,9 @@ import org.mbed.tls.jenkins.BranchInfo 'cc' : 'cc' ] -@Field docker_repo_name = is_open_ci_env ? 'ci-amd64-mbed-tls-ubuntu' : 'jenkins-mbedtls' -@Field docker_ecr = is_open_ci_env ? "trustedfirmware" : "666618195821.dkr.ecr.eu-west-1.amazonaws.com" -@Field docker_repo = "$docker_ecr/$docker_repo_name" +@Field final String docker_repo_name = (is_open_ci_env || is_new_ci_env) ? 'docker.io/trustedfirmware/ci-amd64-mbed-tls-ubuntu' : 'jenkins-mbedtls' +@Field final String docker_ecr = is_new_ci_env ? env.PRIVATE_CONTAINER_REGISTRY : '666618195821.dkr.ecr.eu-west-1.amazonaws.com' +@Field final String docker_repo = is_open_ci_env ? docker_repo_name : "$docker_ecr/$docker_repo_name" /* List of Linux platforms. When a job can run on multiple Linux platforms, * it runs on the first element of the list that supports this job. */ @@ -91,6 +96,16 @@ import org.mbed.tls.jenkins.BranchInfo * Populated by init_docker_images() / gen_jobs.gen_dockerfile_builder_job(platform). */ @Field static docker_tags = [:] +/** Launch an executor with any adjustments that are required by the current CI + * + * @param label The "node label" as used everywhere else in our CI scripts + * @param body A closure to run in the context of the executor + * @return The return value of the closure + */ +def T mbedtls_node(String label, Closure body) { + return node(is_new_ci_env ? "mbedtls-$label" : label, body) +} + /* Compute the git object ID of the Dockerfile. * Equivalent to the `git hash-object ` command. */ @NonCPS @@ -196,7 +211,7 @@ docker pull $docker_repo:$docker_image """ else sh """\ -aws ecr get-login-password | docker login --username AWS --password-stdin $docker_ecr +aws ecr get-login-password --region eu-west-1 | docker login --username AWS --password-stdin $docker_ecr docker pull $docker_repo:$docker_image """ break @@ -254,7 +269,7 @@ List get_branch_information(Collection tls_branches, Collect } list_components_jobs << gen_jobs.job(info.prefix + 'all-platforms') { - node('container-host') { + mbedtls_node('container-host') { try { // Log the environment for debugging purposes sh script: 'export' @@ -312,7 +327,7 @@ List get_branch_information(Collection tls_branches, Collect linux_platforms.each { platform -> list_components_jobs << gen_jobs.job(info.prefix + platform) { - node(gen_jobs.node_label_for_platform(platform)) { + mbedtls_node(gen_jobs.node_label_for_platform(platform)) { try { dir('src') { deleteDir() @@ -416,9 +431,8 @@ void maybe_notify_github(String state, String description, String context=null) } if (context == null) { - def ci = is_open_ci_env ? 'TF OpenCI' : 'Internal CI' def job = env.BRANCH_NAME ==~ /PR-\d+-merge/ ? 'Interface stability tests' : 'PR tests' - context = "$ci: $job" + context = "$ci_name: $job" } githubNotify context: context, @@ -470,8 +484,7 @@ Logs: ${env.BUILD_URL} """ recipients = env.TEST_PASS_EMAIL_ADDRESS } - String subject = ((is_open_ci_env ? "TF Open CI" : "Internal CI") + " ${name} " + \ - (failed ? "failed" : "passed") + "! (branches: ${branches})") + String subject = "$ci_name $name ${failed ? 'failed' : 'passed'}! (branches: ${branches})" echo """\ To: $recipients Subject: $subject @@ -488,7 +501,7 @@ $emailbody @NonCPS boolean pr_author_has_write_access(String repo_name, int pr) { - String credentials = is_open_ci_env ? 'mbedtls-github-token' : 'd015f9b1-4800-4a81-86b3-9dbadc18ee00' + String credentials = (is_open_ci_env || is_new_ci_env) ? 'mbedtls-github-token' : 'd015f9b1-4800-4a81-86b3-9dbadc18ee00' def github = Connector.connect(null, Connector.lookupScanCredentials(currentBuild.rawBuild.parent, null, credentials)) def repo = github.getRepository(repo_name) return repo.getPermission(repo.getPullRequest(pr).user) in [GHPermissionType.ADMIN, GHPermissionType.WRITE] diff --git a/vars/environ.groovy b/vars/environ.groovy index c2a3a9f57..7e10c46fe 100644 --- a/vars/environ.groovy +++ b/vars/environ.groovy @@ -81,7 +81,7 @@ def set_common_pr_production_environment() { /* The credentials here are the SSH credentials for accessing the repositories. They are defined at {JENKINS_URL}/credentials This is a temporary workaround, this should really be set in the Jenkins job configs */ - env.GIT_CREDENTIALS_ID = common.is_open_ci_env ? "mbedtls-github-ssh" : "742b7080-e1cc-41c6-bf55-efb72013bc28" + env.GIT_CREDENTIALS_ID = (common.is_open_ci_env || common.is_new_ci_env) ? "mbedtls-github-ssh" : "742b7080-e1cc-41c6-bf55-efb72013bc28" if (env.BRANCH_NAME ==~ /PR-\d+-merge/) { env.RUN_ABI_CHECK = 'true' } else { diff --git a/vars/gen_jobs.groovy b/vars/gen_jobs.groovy index 9425c48a2..519beb6e1 100644 --- a/vars/gen_jobs.groovy +++ b/vars/gen_jobs.groovy @@ -278,7 +278,7 @@ ${extra_setup_code} try { if (use_docker) { analysis.record_inner_timestamps(node_label, job_name) { - if (common.is_open_ci_env && platform.startsWith('arm-compilers')) { + if ((common.is_open_ci_env || common.is_new_ci_env) && platform.startsWith('arm-compilers')) { withCredentials([string(credentialsId: 'MBEDTLS_ARMCLANG_UBL_CODE', variable:'MBEDTLS_ARMCLANG_UBL_CODE')]) { sh common.docker_script( platform, @@ -704,10 +704,10 @@ def gen_dockerfile_builder_job(String platform, boolean overwrite=false) { def tag = "$image-${common.git_hash_object(dockerfile)}-$arch" def cache = "$image-cache-$arch" def check_docker_image - if (common.is_open_ci_env) { - check_docker_image = "docker manifest inspect $common.docker_repo:$tag > /dev/null 2>&1" + if (common.is_open_ci_env || common.is_new_ci_env) { + check_docker_image = "docker manifest inspect $common.docker_repo:$tag >/dev/null" } else { - check_docker_image = "aws ecr describe-images --repository-name $common.docker_repo_name --image-ids imageTag=$tag" + check_docker_image = "aws ecr describe-images --region eu-west-1 --repository-name $common.docker_repo --image-ids imageTag=$tag" } common.docker_tags[platform] = tag @@ -715,6 +715,30 @@ def gen_dockerfile_builder_job(String platform, boolean overwrite=false) { return job(platform) { def node_label = arch == 'amd64' ? 'dockerfile-builder' : "container-host-$arch" analysis.node_record_timestamps(node_label, platform) { + if (common.is_open_ci_env || common.is_new_ci_env) { + withCredentials([string(credentialsId: 'DOCKER_AUTH', variable: 'TOKEN')]) { + sh """\ +mkdir -p ${env.HOME}/.docker +cat > ${env.HOME}/.docker/config.json << EOF +{ + "auths": { + "https://index.docker.io/v1/": { + "auth": "\${TOKEN}" + } + } +} +EOF +chmod 0600 ${env.HOME}/.docker/config.json +""" + } + } + + if (!common.is_open_ci_env) { + sh """\ +aws ecr get-login-password --region eu-west-1 | docker login --username AWS --password-stdin $common.docker_ecr +""" + } + /* Take the lock only once we are running on a node. * This prevents a low-priority job from hogging the lock, when a high-priority job (eg. a merge queue job) * is added to the queue later. @@ -734,34 +758,20 @@ def gen_dockerfile_builder_job(String platform, boolean overwrite=false) { writeFile file: 'Dockerfile', text: dockerfile def extra_build_args = '' - if (common.is_open_ci_env) { - withCredentials([string(credentialsId: 'DOCKER_AUTH', variable: 'TOKEN')]) { - sh """\ -mkdir -p ${env.HOME}/.docker -cat > ${env.HOME}/.docker/config.json << EOF -{ - "auths": { - "https://index.docker.io/v1/": { - "auth": "\${TOKEN}" - } - } -} -EOF -chmod 0600 ${env.HOME}/.docker/config.json -""" - } - } else { - sh """\ -aws ecr get-login-password | docker login --username AWS --password-stdin $common.docker_ecr -""" - } - // Generate download URL for armclang if (platform.startsWith('arm-compilers')) { - withCredentials(common.is_open_ci_env ? [] : [aws(credentialsId: 'armclang-readonly-keys')]) { - sh ''' -aws s3 presign s3://trustedfirmware-private/armclang/ARMCompiler6.21_standalone_linux-x86_64.tar.gz >armc6_url -''' + withCredentials((common.is_open_ci_env || common.is_new_ci_env) ? [] : [aws(credentialsId: 'armclang-readonly-keys')]) { + final String bucket, region + if (common.is_new_ci_env) { + bucket = "openci-trustedfirmware-private-$env.INFRA_ENV" + region = 'eu-west-1' + } else { + bucket = 'trustedfirmware-private' + region = 'us-east-1' + } + sh """\ +aws s3 presign --region $region s3://$bucket/armclang/ARMCompiler6.21_standalone_linux-x86_64.tar.gz >armc6_url +""" extra_build_args += ' --secret id=armc6_url,src=./armc6_url' } diff --git a/vars/mbedtls.groovy b/vars/mbedtls.groovy index a7ae6acac..9d6abcab4 100644 --- a/vars/mbedtls.groovy +++ b/vars/mbedtls.groovy @@ -106,7 +106,7 @@ void run_pr_job(String target_repo, boolean is_production, Collection tl try { common.maybe_notify_github('PENDING', 'In progress') - if (common.is_open_ci_env && is_merge_queue) { + if (is_merge_queue) { // Fake required checks that don't run in the merge queue def skipped_checks = [ 'DCO',