From 88f6531098f37b86e4e25af6396b746013662231 Mon Sep 17 00:00:00 2001 From: cornelia247 Date: Wed, 12 Oct 2022 17:42:30 +0100 Subject: [PATCH 1/5] fixed the "build all pass" typo to "builds all pass" --- docs/diagram.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/diagram.md b/docs/diagram.md index 621cb95d3..2369d10db 100644 --- a/docs/diagram.md +++ b/docs/diagram.md @@ -387,7 +387,7 @@ job[openjdk_build_docker_multiarch] --load--> load[Jenkinsfile] --stage1--> stage1[Docker build stage] -stage1 --> 11[linux x64] --> stagecheck{ build all pass} +stage1 --> 11[linux x64] --> stagecheck{ builds all pass} stage1 --> 12[linux aarch64]--> stagecheck From b21c9868d80cc5661de1ebf671120e55a0d428a7 Mon Sep 17 00:00:00 2001 From: cornelia247 Date: Thu, 13 Oct 2022 20:32:26 +0100 Subject: [PATCH 2/5] parementalised Sign_node --- pipelines/build/common/openjdk_build_pipeline.groovy | 7 +++++-- pipelines/defaults.json | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy index 9b28fb564..b7a568be2 100644 --- a/pipelines/build/common/openjdk_build_pipeline.groovy +++ b/pipelines/build/common/openjdk_build_pipeline.groovy @@ -485,6 +485,7 @@ class Build { Run the Sign downstream job. We run this job on windows and jdk8 hotspot & jdk13 mac builds. The job code signs and notarizes the binaries so they can run on these operating systems without encountering issues. */ + def SIGN_NODE_LABEL = DEFAULTS_JSON['signNodeLabel'] def sign(VersionInfo versionInfo) { // Sign and archive jobs if needed if ( @@ -493,7 +494,8 @@ class Build { context.stage('sign') { def filter = '' - def nodeFilter = 'eclipse-codesign' + def nodeFilter = "${SIGN_NODE_LABEL}" + if (buildConfig.TARGET_OS == 'windows') { filter = '**/OpenJDK*_windows_*.zip' @@ -701,7 +703,8 @@ class Build { default: break } - def nodeFilter = 'eclipse-codesign' + /* groovylint-disable-next-line VariableTypeRequired */ + def nodeFilter = "${SIGN_NODE_LABEL}" // Execute sign installer job def installerJob = context.build job: 'build-scripts/release/sign_installer', diff --git a/pipelines/defaults.json b/pipelines/defaults.json index 759dcc31f..449e476d5 100644 --- a/pipelines/defaults.json +++ b/pipelines/defaults.json @@ -53,5 +53,6 @@ "numMachines" : ["3"] } }, - "defaultsUrl" : "https://raw.githubusercontent.com/adoptium/ci-jenkins-pipelines/master/pipelines/defaults.json" + "defaultsUrl" : "https://raw.githubusercontent.com/adoptium/ci-jenkins-pipelines/master/pipelines/defaults.json", + "signNodeLabel" : "eclipse-codesign" } From 90d6e87de7faf637396d6809ac643d3bbed54908 Mon Sep 17 00:00:00 2001 From: Cornelia Ekpenyong <59790379+cornelia247@users.noreply.github.com> Date: Thu, 13 Oct 2022 21:29:21 +0100 Subject: [PATCH 3/5] Update pipelines/defaults.json Co-authored-by: Martijn Verburg --- pipelines/defaults.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/defaults.json b/pipelines/defaults.json index 4432b2791..20a2ed517 100644 --- a/pipelines/defaults.json +++ b/pipelines/defaults.json @@ -54,5 +54,5 @@ } }, "defaultsUrl" : "https://raw.githubusercontent.com/adoptium/ci-jenkins-pipelines/master/pipelines/defaults.json", - "signNodeLabel" : "eclipse-codesign" + "signNodeLabel" : "eclipse-codesign" } From 92bb465cca2c419bbab23341c2d3b26bbc85fefa Mon Sep 17 00:00:00 2001 From: cornelia247 Date: Thu, 13 Oct 2022 21:38:14 +0100 Subject: [PATCH 4/5] fixed README.md issue --- CONTRIBUTING.md | 6 +++--- pipelines/build/regeneration/README.md | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 176b73b09..ac9064fdc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,7 @@ You can contribute to this project in many different ways. **We appreciate all ## Project description -ci-jenkins-pipelines is a project that contains the groovy code that powers our jenkins ci build system. +ci-jenkins-pipelines is a project that contains the groovy code that powers our jenkins ci build tool. * @@ -39,7 +39,7 @@ Contact the Eclipse Foundation Webdev team via webdev@eclipse-foundation.org. Please let us know via our [issue tracker](https://github.com/adoptium/temurin-build/issues) if you find a problem, even if you don't have a fix for it. The ideal issue report should be descriptive, and where possible include the steps we can take to reproduce the problem for ourselves. -If you have a proposed fix for an issue, or an enhancement you would like to make to the code please describe it in an issue, then send us the code as a [Github pull request](https://help.github.com/articles/about-pull-requests) as described below. +If you have a proposed fix for an issue, or an enhancement you would like to make to the code please describe it in an issue, then send us the code as a [GitHub pull request](https://help.github.com/articles/about-pull-requests) as described below. ## Pull requests @@ -56,7 +56,7 @@ Use a descriptive title, and if it relates to an issue in our tracker please ref ### Pull request comment -The PR comment should be formed by a one-line subject, followed by one line of white space, and one or more descriptive paragraphs, each separated by one line of white space. All of them should be finished by a dot. +The PR comment should be formed by a one-line subject, followed by one line of whitespace, and one or more descriptive paragraphs, each separated by one line of whitespace. All of them should be finished by a dot. Where your PR fixes an issue, it should include a reference to the issue's identifier in the first line of the commit comment. The comment should provide enough information for a reviewer to understand the changes and their relation to the rest of the code. diff --git a/pipelines/build/regeneration/README.md b/pipelines/build/regeneration/README.md index 23eb20e69..6b1db1dfd 100644 --- a/pipelines/build/regeneration/README.md +++ b/pipelines/build/regeneration/README.md @@ -8,10 +8,10 @@ To enable concurrent pipeline builds (i.e. submitting two pipelines in parallel) All of our [pipelines](https://ci.adoptopenjdk.net/job/build-scripts/) make use of [downstream jobs](https://ci.adoptopenjdk.net/job/build-scripts/job/jobs/) to build Adopts JDK's. In short, the jobs are created in the pipelines with a set of configurations passed down to them. To create these jobs, we utilise a plugin called [job dsl](https://github.com/jenkinsci/job-dsl-plugin) to create a `dsl` file for each downstream job, containing the configurations, node label, etc. -In the past, we created these dsl's in the [pipeline files](pipelines/build). So each time we wanted to create a downstream job, we would create all of the job dsl's that were possible for the pipeline and pick the one that we needed. +In the past, we created these dsl's in the [pipeline files](/pipelines/build). So each time we wanted to create a downstream job, we would create all of the job dsl's that were possible for the pipeline and pick the one that we needed. Not only was this resource intensive and slow, but it also meant that concurrent builds were impossible due to the risk of one builds dsl's overwriting another's. This is why we have created pipeline job generators to create the dsl's for the pipelines to use, instead of creating them in the pipeline jobs. -The job regenerators are essentially downstream job makers. They pull in the [targetConfigurations](pipelines/jobs/configurations) and build the job DSL's for each possible downstream job. +The job regenerators are essentially downstream job makers. They pull in the [targetConfigurations](/pipelines/jobs/configurations) and build the job DSL's for each possible downstream job. The pipelines can use these job dsl's to create their downstream jobs since they are created in the same node as them (the master one). This way, each of the pipelines has a fresh dsl each time, no matter how many builds are running at once. ## Where they are @@ -24,12 +24,12 @@ NOTE: When the JDK HEAD updates, these jobs will need to be updated too (see [RE There are three stages for each job regenerator. - Execute the top level job: - - The jobs themselves are executed by GitHub Push on this repository. Each time there is a commit, all the pipeline regenerators are kicked off. This is so any potential changes to the [buildConfigurations](pipelines/jobs/configurations/jdk8u_pipeline_config.groovy) and [targetConfigurations](pipelines/jobs/configurations/jdk8u.groovy) are taken into account when creating a job dsl for each downstream job. - - Each of the jobs executes it's corresponding [regeneration](pipelines/build/regeneration) file, passing down it's version, targeted OS/ARCH/VARIANT and specific build configurations to the main [config_regeneration](pipelines/build/common/config_regeneration.groovy) file. + - The jobs themselves are executed by GitHub Push on this repository. Each time there is a commit, all the pipeline regenerators are kicked off. This is so any potential changes to the [buildConfigurations](/pipelines/jobs/configurations/jdk8u_pipeline_config.groovy) and [targetConfigurations](/pipelines/jobs/configurations/jdk8u.groovy) are taken into account when creating a job dsl for each downstream job. + - Each of the jobs executes it's corresponding [regeneration](/pipelines/build/regeneration) file, passing down it's version, targeted OS/ARCH/VARIANT and specific build configurations to the main [config_regeneration](/pipelines/build/common/config_regeneration.groovy) file. - Check if the corresponding pipeline is in progress: - Since we want to potentially avoid overwriting the job dsl's of any pipelines in progress, we use the [jenkins API](https://ci.adoptopenjdk.net/api/) to verify that there are no pipelines of that version queued or running. If there are, the job regenerator sleeps for 15mins and checks again afterwards. If not, it moves onto the next step. - Regenerate the downstream jobs, one at a time: - - The regenerator then iterates through the keys in the `targetConfigurations` (e.g. [jdk11u.groovy](pipelines/jobs/configurations/jdk11u.groovy)), which are the same keys used in the `buildConfiguration` file. + - The regenerator then iterates through the keys in the `targetConfigurations` (e.g. [jdk11u.groovy](/pipelines/jobs/configurations/jdk11u.groovy)), which are the same keys used in the `buildConfiguration` file. After parsing each variant in them and going through various error handling stages, the job name and folder path is constructed which is the bare minimum that the job dsl needs to be created. We only need the bare minimum as the pipelines will overwrite most the configs when they run. - The job dsl for that downstream job is constructed and that job is then, successfully regenerated. The result is somewhat similar to this: @@ -90,7 +90,7 @@ Unreferenced items: ## Build Pipeline Generator This generator generates the [top level](https://ci.adoptopenjdk.net/job/build-scripts/) pipeline jobs. It works by iterating through the config files, defining a job dsl configuration for each version that has a version config file. -It then calls [pipeline_job_template.groovy](pipelines/jobs/pipeline_job_template.groovy) to finalise the dsl. By default, the [job that runs this file](https://ci.adoptopenjdk.net/job/build-scripts/job/utils/job/build-pipeline-generator/) has restricted read access so you will likely need to contact a jenkins admin to see the results of the job. +It then calls [pipeline_job_template.groovy](/pipelines/jobs/pipeline_job_template.groovy) to finalise the dsl. By default, the [job that runs this file](https://ci.adoptopenjdk.net/job/build-scripts/job/utils/job/build-pipeline-generator/) has restricted read access so you will likely need to contact a jenkins admin to see the results of the job. ## Downstream Test Jobs From 2fbe888a12e1b9f805859ddee978bc5983da779b Mon Sep 17 00:00:00 2001 From: cornelia247 Date: Thu, 13 Oct 2022 21:42:37 +0100 Subject: [PATCH 5/5] removed space --- pipelines/defaults.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/defaults.json b/pipelines/defaults.json index 20a2ed517..44543070a 100644 --- a/pipelines/defaults.json +++ b/pipelines/defaults.json @@ -54,5 +54,5 @@ } }, "defaultsUrl" : "https://raw.githubusercontent.com/adoptium/ci-jenkins-pipelines/master/pipelines/defaults.json", - "signNodeLabel" : "eclipse-codesign" + "signNodeLabel" : "eclipse-codesign" }