From a9f64a46ecc8b03bdb34a7ae32263c28d584d539 Mon Sep 17 00:00:00 2001 From: Hannes Wellmann Date: Sat, 19 Apr 2025 18:57:58 +0200 Subject: [PATCH] [Build] Migrate Promotion and Make-Visble jobs to Jenkins pipeline and clean them up. --- JenkinsJobs/Releng/FOLDER.groovy | 92 +++++++++++++++++- JenkinsJobs/Releng/makeVisible.groovy | 64 ------------ JenkinsJobs/Releng/makeVisible.jenkinsfile | 25 +++++ JenkinsJobs/Releng/renameAndPromote.groovy | 75 -------------- .../Releng/renameAndPromote.jenkinsfile | 34 +++++++ JenkinsJobs/Releng/tagEclipseRelease.groovy | 97 ------------------- .../Releng/tagEclipseRelease.jenkinsfile | 56 +++++++++++ RELEASE.md | 17 ++-- 8 files changed, 215 insertions(+), 245 deletions(-) delete mode 100644 JenkinsJobs/Releng/makeVisible.groovy create mode 100644 JenkinsJobs/Releng/makeVisible.jenkinsfile delete mode 100644 JenkinsJobs/Releng/renameAndPromote.groovy create mode 100644 JenkinsJobs/Releng/renameAndPromote.jenkinsfile delete mode 100644 JenkinsJobs/Releng/tagEclipseRelease.groovy create mode 100644 JenkinsJobs/Releng/tagEclipseRelease.jenkinsfile diff --git a/JenkinsJobs/Releng/FOLDER.groovy b/JenkinsJobs/Releng/FOLDER.groovy index 652686d90e0..a07865f7eaa 100644 --- a/JenkinsJobs/Releng/FOLDER.groovy +++ b/JenkinsJobs/Releng/FOLDER.groovy @@ -1,5 +1,5 @@ folder('Releng') { - description('Jobs related to routine releng tasks. Some are periodic, some are "manual" jobs ran only when needed.') + description('Jobs related to routine releng tasks. Some are periodic, some are "manual" jobs ran only when needed.') } pipelineJob('Releng/PublishToMaven'){ @@ -48,7 +48,7 @@ Snapshots are published to Maven central by publishing to a staging repository. - ''') +''') } definition { cpsScm { @@ -86,3 +86,91 @@ pipelineJob('Releng/prepareNextDevCycle'){ } } +pipelineJob('Releng/renameAndPromote'){ + displayName('Rename and Promote') + description('''\ +This job does the "stage 1" or first part of a promotion. +It renames the files for Equinox and Eclipse, creates an appropriate repo on 'downloads', rsync's everything to 'downloads', but leave everything "invisible" -- unless someone knows the exact URL. +This allows two things. First, allows artifacts some time to "mirror" when that is needed. +But also, allows the sites and repositories to be examined for correctness before making them visible to the world. +The second (deferred) step that makes things visible works, in part, based on some output of this first step. Hence, they must "share a workspace". +''') + parameters { + stringParam('DROP_ID', null, '''\ +The name (or, build id) of the build to rename and promote. Typically would be a value such as I20160530-2000 or M20160912-1000. +It must match the name of the build on the build machine. + ''') + stringParam('CHECKPOINT', null, 'M1, M3, RC1, RC2, RC3 etc (blank for final releases).') + stringParam('SIGNOFF_BUG', null, 'The issue that was used to "signoff" the checkpoint. If there are no unit test failures, this can be left blank. Otherwise a link is added to test page explaining that "failing unit tests have been investigated".') + stringParam('TRAIN_NAME', null, 'The name of the release stream, typically yyyy-mm. For example: 2022-09') + stringParam('STREAM', null, 'Needs to be all three files of primary version for the release, such as 4.7.1 or 4.8.0.') + stringParam('DL_TYPE', null, "This is the build type we are promoting TO. I-builds promote to 'S' until 'R'.") + stringParam('TAG', null, ''' For passing to the tagEclipseRelease job. +R is used for release builds. For example: R4_25 +S is used for milestones and includes the milestone version. For example: S4_25_0_RC2 +''') + } + definition { + cpsScm { + lightweight(true) + scm { + github('eclipse-platform/eclipse.platform.releng.aggregator', 'master') + } + scriptPath('JenkinsJobs/Releng/renameAndPromote.jenkinsfile') + } + } +} + +pipelineJob('Releng/tagEclipseRelease'){ + displayName('Tag Eclipse Release') + description('Tag promoted builds.') + parameters { + stringParam('tag', null, '''\ +R is used for release builds. For example: R4_25 +S is used for milestones and includes the milestone version. For example: S4_25_0_RC2 +''') + stringParam('buildID', null, 'I-build ID of the build that was promoted, for example: I20220831-1800') + stringParam('annotation', null, '''\ +GitHub issue to track tagging the release, for example: +'https://github.com/eclipse-platform/eclipse.platform.releng.aggregator/issues/3058' - Tag Eclipse 4.36 release +''') + } + definition { + cpsScm { + lightweight(true) + scm { + github('eclipse-platform/eclipse.platform.releng.aggregator', 'master') + } + scriptPath('JenkinsJobs/Releng/tagEclipseRelease.jenkinsfile') + } + } +} + +pipelineJob('Releng/makeVisible'){ + displayName('Make Visible') + description('''\ +The first part of doing a promotion -- the Rename And Promote job -- creates some scripts that make this deferred "make visible" part possible. +Therefore, they have to share a 'workspace', and the output of the first job must remain in place until its time to "make visible". +''') + parameters { + stringParam('DROP_ID', null, '''\ +The name (or, build id) of the build to rename and promote. Typically would be a value such as I20160530-2000 or M20160912-1000. +It must match the name of the build on the build machine. +''') + stringParam('CHECKPOINT', null, 'M1, M3, RC1, RC2, RC3 etc (blank for final releases).') + stringParam('SIGNOFF_BUG', null, 'The issue that was used to "signoff" the checkpoint. If there are no unit test failures, this can be left blank. Otherwise a link is added to test page explaining that "failing unit tests have been investigated".') + stringParam('TRAIN_NAME', null, 'The name of the release stream, typically yyyy-mm. For example: 2022-09') + stringParam('STREAM', null, 'Needs to be all three files of primary version for the release, such as 4.7.1 or 4.8.0.') + stringParam('DL_TYPE', null, "This is the build type we are promoting TO. I-builds promote to 'S' until 'R'.") + } + definition { + cpsScm { + lightweight(true) + scm { + github('eclipse-platform/eclipse.platform.releng.aggregator', 'master') + } + scriptPath('JenkinsJobs/Releng/makeVisible.jenkinsfile') + } + } +} + diff --git a/JenkinsJobs/Releng/makeVisible.groovy b/JenkinsJobs/Releng/makeVisible.groovy deleted file mode 100644 index a0da2c83d60..00000000000 --- a/JenkinsJobs/Releng/makeVisible.groovy +++ /dev/null @@ -1,64 +0,0 @@ -job('Releng/makeVisible'){ - displayName('Make Visible') - description(''' -The first part of doing a promotion -- the Rename And Promote job -- creates some scripts that make this deferred "make visible" part possible. - -Therefore, they have to share a workspace, and the output of the first job must remain in place until its time to "make visible". - - ''') - - parameters { - stringParam('DROP_ID', null, ''' -The name (or, build id) of the build to rename and promote. Typically would be a value such as I20160530-2000 or M20160912-1000. -It must match the name of the build on the build machine. - ''') - stringParam('CHECKPOINT', null, 'M1, M3, RC1, RC2, RC3 etc (blank for final releases).') - stringParam('SIGNOFF_BUG', null, 'The issue that was used to "signoff" the checkpoint. If there are no unit test failures, this can be left blank. Otherwise a link is added to test page explaining that "failing unit tests have been investigated".') - stringParam('TRAIN_NAME', null, 'The name of the release stream, typically yyyy-mm. For example: 2022-09') - stringParam('STREAM', null, 'Needs to be all three files of primary version for the release, such as 4.7.1 or 4.8.0.') - stringParam('DL_TYPE', null, ''' -This is the build type we are promoting TO. -I-builds promote to 'S' until 'R'. - ''') - } - - logRotator { - daysToKeep(10) - } - - jdk('adoptopenjdk-hotspot-jdk11-latest') - - label('basic') - - wrappers { //adds pre/post actions - timestamps() - sshAgent('projects-storage.eclipse.org-bot-ssh', 'git.eclipse.org-bot-ssh', 'github-bot-ssh') - timeout { - absolute(60) - } - } - - steps { - shell(''' -#!/bin/bash -x - -wget https://download.eclipse.org/eclipse/relengScripts/cje-production/promotion/makeVisible.sh - -chmod +x makeVisible.sh - -${WORKSPACE}/makeVisible.sh - ''') - } - - publishers { - downstreamParameterized { - trigger('Releng/updateIndex') { - triggerWithNoParameters(true) - } - } - extendedEmail { - recipientList("sravankumarl@in.ibm.com") - } - } -} - diff --git a/JenkinsJobs/Releng/makeVisible.jenkinsfile b/JenkinsJobs/Releng/makeVisible.jenkinsfile new file mode 100644 index 00000000000..1281645356a --- /dev/null +++ b/JenkinsJobs/Releng/makeVisible.jenkinsfile @@ -0,0 +1,25 @@ +pipeline { + options { + skipDefaultCheckout() + timestamps() + timeout(time: 15, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr:'5')) + } + agent { + label 'basic' + } + stages { + stage('Make visible') { + steps { + sshagent(['projects-storage.eclipse.org-bot-ssh', 'github-bot-ssh']) { + sh ''' + curl -o makeVisible.sh https://download.eclipse.org/eclipse/relengScripts/cje-production/promotion/makeVisible.sh + chmod +x makeVisible.sh + ./makeVisible.sh + ''' + } + build job: 'Releng/updateIndex', wait: false + } + } + } +} diff --git a/JenkinsJobs/Releng/renameAndPromote.groovy b/JenkinsJobs/Releng/renameAndPromote.groovy deleted file mode 100644 index a56d5f58be3..00000000000 --- a/JenkinsJobs/Releng/renameAndPromote.groovy +++ /dev/null @@ -1,75 +0,0 @@ -job('Releng/renameAndPromote'){ - displayName('Rename and Promote') - description(''' -This job does the "stage 1" or first part of a promotion. It renames the files for Equinox and Eclipse, creates an appropriate repo on "downloads", rsync's everything to 'downloads', but leave everything "invisible" -- unless someone knows the exact URL. This allows two things. First, allows artifacts some time to "mirror" when that is needed. But, also, allows the sites and repositories to be examined for correctness before making them visible to the world. - -The second (deferred) step that makes things visible works, in part, based on some output of this first step. Hence, they must "share a workspace". - ''') - - parameters { - stringParam('DROP_ID', null, ''' -The name (or, build id) of the build to rename and promote. Typically would be a value such as I20160530-2000 or M20160912-1000. -It must match the name of the build on the build machine. - ''') - stringParam('CHECKPOINT', null, 'M1, M3, RC1, RC2, RC3 etc (blank for final releases).') - stringParam('SIGNOFF_BUG', null, 'The issue that was used to "signoff" the checkpoint. If there are no unit test failures, this can be left blank. Otherwise a link is added to test page explaining that "failing unit tests have been investigated".') - stringParam('TRAIN_NAME', null, 'The name of the release stream, typically yyyy-mm. For example: 2022-09') - stringParam('STREAM', null, 'Needs to be all three files of primary version for the release, such as 4.7.1 or 4.8.0.') - stringParam('DL_TYPE', null, ''' -This is the build type we are promoting TO. -I-builds promote to 'S' until 'R'. - ''') - stringParam('TAG', null, ''' For passing to the tagEclipseRelease job. -R is used for release builds. For example: R4_25 -S is used for milestones and includes the milestone version. For example: S4_25_0_RC2 - ''') - } - - logRotator { - daysToKeep(10) - numToKeep(5) - } - - jdk('adoptopenjdk-hotspot-jdk11-latest') - - label('basic') - - wrappers { //adds pre/post actions - timestamps() - sshAgent('projects-storage.eclipse.org-bot-ssh') - timeout { - absolute(60) - } - } - - steps { - shell(''' -#!/bin/bash -x - -cd ${WORKSPACE} -wget https://download.eclipse.org/eclipse/relengScripts/cje-production/promotion/promoteSites.sh - -chmod +x promoteSites.sh - -${WORKSPACE}/promoteSites.sh - ''') - } - - publishers { - downstreamParameterized { - trigger('tagEclipseRelease') { - parameters { - predefinedProp('tag', '$TAG') - predefinedProp('buildID', '$DROP_ID') - predefinedProp('annotation', '$SIGNOFF_BUG') - } - } - } - archiveArtifacts { - pattern('**/stage2output*/**') - } - extendedEmail { - recipientList("sravankumarl@in.ibm.com") - } - } -} \ No newline at end of file diff --git a/JenkinsJobs/Releng/renameAndPromote.jenkinsfile b/JenkinsJobs/Releng/renameAndPromote.jenkinsfile new file mode 100644 index 00000000000..3e87983f03e --- /dev/null +++ b/JenkinsJobs/Releng/renameAndPromote.jenkinsfile @@ -0,0 +1,34 @@ +pipeline { + options { + skipDefaultCheckout() + timestamps() + timeout(time: 120, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr:'5')) + } + agent { + label 'basic' + } + stages { + stage('Rename and Promote') { + steps { + sshagent(['projects-storage.eclipse.org-bot-ssh']) { + sh '''#!/bin/bash -x + curl -o promoteSites.sh https://download.eclipse.org/eclipse/relengScripts/cje-production/promotion/promoteSites.sh + chmod +x promoteSites.sh + ./promoteSites.sh + ''' + } + build job: 'Releng/tagEclipseRelease', wait: true, propagate: true, parameters: [ + string(name: 'tag', value: "${TAG}"), + string(name: 'buildID', value: "${DROP_ID}"), + string(name: 'annotation', value: "${ params.SIGNOFF_BUG ? 'https://github.com/eclipse-platform/eclipse.platform.releng.aggregator/issues/' + params.SIGNOFF_BUG : '' }") + ] + } + } + } + post { + always { + archiveArtifacts '**/stage2output*/**' + } + } +} diff --git a/JenkinsJobs/Releng/tagEclipseRelease.groovy b/JenkinsJobs/Releng/tagEclipseRelease.groovy deleted file mode 100644 index 457ec329dd2..00000000000 --- a/JenkinsJobs/Releng/tagEclipseRelease.groovy +++ /dev/null @@ -1,97 +0,0 @@ -job('Releng/tagEclipseRelease'){ - displayName('Tag Eclipse Release') - description('Tag promoted builds.') - - logRotator { - daysToKeep(5) - numToKeep(5) - } - - parameters { - stringParam('tag', null, ''' -R is used for release builds. For example: R4_25 -S is used for milestones and includes the milestone version. For example: S4_25_0_RC2 - ''') - stringParam('buildID', null, 'I-build ID of the build that was promoted, for example: I20220831-1800') - stringParam('annotation', null, ''' -GitHub issue (or legacy bugzilla bug ID) to track tagging the release, for example: -#557 - Tag Eclipse 4.25 release - ''') - } - - jdk('oracle-jdk8-latest') - - label('basic') - - wrappers { //adds pre/post actions - timestamps() - sshAgent('git.eclipse.org-bot-ssh', 'github-bot-ssh') - preBuildCleanup() - } - - steps { - shell(''' -#!/bin/bash - -# Strip spaces from the buildID and tag -buildID=$(echo $buildID|tr -d ' ') -tag=$(echo $tag|tr -d ' ') - -#If build id or tag is empty we need to exit. -if [ -z "$buildID" ] -then - exit 1 -fi - -if [ -z "$tag" ] -then - exit 1 -fi - -buildType=$(echo ${buildID}|cut -c 1) - -cd ${WORKSPACE} - -git config --global user.email "releng-bot@eclipse.org" -git config --global user.name "Eclipse Releng Bot" -git clone -b master --recursive git@github.com:eclipse-platform/eclipse.platform.releng.aggregator.git - -cd eclipse.platform.releng.aggregator - -function toPushRepo() { - from="$1" - if ! [[ "$from" == http* ]]; then - echo $from - else - echo $(sed -e 's/https:\\/\\/github.com/ssh:\\/\\/git@github.com/g' <<< $from) - fi -} - -tags=$(git tag -l) -if [[ ! $(echo $tags | grep $tag) ]]; then { - git tag -a -m "${annotation}" ${tag} ${buildID} - git push --verbose $(toPushRepo $(git config --get remote.origin.url)) tag ${tag} -} else { - echo "Already tagged repo $(toPushRepo $(git config --get remote.origin.url))" -} -fi - -for i in $(ls) -do - if [[ -d $i ]]; then { - pushd $i - tags=$(git tag -l) - if [[ ! $(echo $tags | grep $tag) ]]; then { - git tag -a -m "${annotation}" ${tag} ${buildID} - git push --verbose $(toPushRepo $(git config --get remote.origin.url)) tag ${tag} - } else { - echo "Already tagged repo $(toPushRepo $(git config --get remote.origin.url))" - } - fi - popd - } - fi -done - ''') - } -} diff --git a/JenkinsJobs/Releng/tagEclipseRelease.jenkinsfile b/JenkinsJobs/Releng/tagEclipseRelease.jenkinsfile new file mode 100644 index 00000000000..489a9383040 --- /dev/null +++ b/JenkinsJobs/Releng/tagEclipseRelease.jenkinsfile @@ -0,0 +1,56 @@ +pipeline { + options { + timestamps() + timeout(time: 15, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr:'5')) + } + agent { + label 'basic' + } + stages { + stage('Checkout Submodules') { + steps { + sh ''' + git fetch origin tag "${buildID}" + git checkout "${buildID}" + # Check out all submodules at the specified buildID + git submodule update --init --recursive --depth 1 + + git config --global user.email 'releng-bot@eclipse.org' + git config --global user.name 'Eclipse Releng Bot' + ''' + } + } + stage('Tag build') { + steps { + sshagent(['github-bot-ssh']) { + sh '''#!/bin/bash -xe + # Strip spaces from the buildID and tag + buildID=$(echo $buildID|tr -d ' ') + tag=$(echo $tag|tr -d ' ') + + #If build id or tag is empty we need to exit. + if [ -z "$buildID" ] || [ -z "$tag" ]; then + echo 'buildID or tag input is not specified' + exit 1 + fi + + function tagBuild() { + pushURL=$(git config remote.origin.url) + if [[ "$pushURL" == http* ]]; then + # Change to SSH, if the configured URL uses HTTPS (we can only push with SSH) + pushURL=$(echo $pushURL|sed --expression 's|https://github.com/|git@github.com:|') + fi + git tag -a -m "${annotation}" ${tag} HEAD + # git push fails if the tag already exists + git push --verbose ${pushURL} tag ${tag} + } + tagBuild + export -f tagBuild + git submodule foreach 'tagBuild' + ''' + } + } + } + } +} diff --git a/RELEASE.md b/RELEASE.md index e2d34bfb475..0f801ee6204 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -18,7 +18,10 @@ ### Milestone/RC Week - **M 1/2/3 Release** - * All milestone releases are 'lightweight', meaning there is no announcement or signoff. No additional builds need to be run, just the daily I-build at 6PM EST. Thursdays build is promoted to simrel on friday (unless there are problems with Thursdays build, in which case promote Wednesdays) and the compiler is updated if necessary, but the promote and makevisible jobs don't need to be run. + * All milestone releases are 'lightweight', meaning there is no announcement or signoff. + No additional builds need to be run, just the daily I-build at 6PM EST. + Thursdays build is promoted to simrel on friday (unless there are problems with Thursdays build, in which case promote Wednesdays) and the compiler is updated if necessary, + but the `Promote` and `Make Visible` jobs don't need to be run. - **Wednesday**: * Verify that EMF, ECF and Orbit contributions have been included (if applicable). * Final release candidate build runs at 6PM EST. @@ -32,7 +35,7 @@ * Just [1 line](https://www.eclipse.org/lists/platform-releng-dev/msg38086.html) asking for sign off on the GitHub issue created in the previous step. - **Friday**: * **Promote** the release candidate (if go). - * Run the [rename and promote](https://ci.eclipse.org/releng/job/Releng/job/renameAndPromote/) job in Jenkins + * Run the [Rename and Promote](https://ci.eclipse.org/releng/job/Releng/job/renameAndPromote/) job in Jenkins - DROP_ID: Release candidate build ID (make sure there is no space before or after the ID). - CHECKPOINT: M1 etc (blank for final releases) - SIGNOFF_BUG: Needs to be updated to sign-off issue (numeric part only) @@ -40,7 +43,7 @@ - STREAM: 4.24.0 etc - DL_TYPE: S is used to promote I-builds. - TAG: Parameter should match stream version, i.e `S4_30_0_RC1` etc - - After the build find and open the mail template [artifact](https://ci.eclipse.org/releng/job/eclipse.releng.renameAndPromote/lastSuccessfulBuild/artifact/) and have it ready. + - After the build find and open the mail template [artifact](https://ci.eclipse.org/releng/job/Releng/job/renameAndPromote/lastSuccessfulBuild/artifact/) and have it ready. - This should automatically run [tag Eclipse release](https://ci.eclipse.org/releng/job/Releng/job/tagEclipseRelease/) to tag the source code. * Contribute to SimRel - If you have not already set up SimRel you can do so using Auto Launch [here](https://www.eclipse.org/setups/installer/?url=https://git.eclipse.org/c/oomph/org.eclipse.oomph.git/plain/setups/interim/SimultaneousReleaseTrainConfiguration.setup&show=true) @@ -52,10 +55,10 @@ 5. Commit Simrel updates to Gerrit - Message should use year-month format, i.e "Simrel updates for Eclipse and Equinox for 2022-06 M1" * Make the build visible - - Run the [make visible](https://ci.eclipse.org/releng/job/Releng/job/makeVisible/) job in Releng jenkins to make the promoted build visible on the download page. - - Parameters should match Rename and Promote job + - Run the [Make Visible](https://ci.eclipse.org/releng/job/Releng/job/makeVisible/) job in Releng jenkins to make the promoted build visible on the download page. + - Parameters should match `Rename and Promote` job * Send email that the M1 build is available - - Use the mail template from the promotion build [artifacts](https://ci.eclipse.org/releng/job/eclipse.releng.renameAndPromote/lastSuccessfulBuild/artifact/) in Jenkins to get the download urls. + - Use the mail template from the promotion build [artifacts](https://ci.eclipse.org/releng/job/Releng/job/renameAndPromote/lastSuccessfulBuild/artifact/) in Jenkins to get the download urls. - Make sure to mention that the Master branch is now again open for development. * For **Milestone builds** return the I-builds to the normal schedule. * **After RC1** @@ -102,7 +105,7 @@ The actual steps to release **Friday** * #### **Promote to GA** - - After Simrel declares RC2 (usually the Friday before release) run the [rename and promote](https://ci.eclipse.org/releng/job/eclipse.releng.renameAndPromote/) job to promote RC2 (or RC2a). If the [daily cleanup for old builds](https://ci.eclipse.org/releng/job/Cleanup/job/dailyCleanOldBuilds/) job was not disabled and the original I-build is no longer available you can use the promoted RC2 build. + - After Simrel declares RC2 (usually the Friday before release) run the [Rename and Promote](https://ci.eclipse.org/releng/job/Releng/job/renameAndPromote/) job to promote RC2 (or RC2a). If the [daily cleanup for old builds](https://ci.eclipse.org/releng/job/Cleanup/job/dailyCleanOldBuilds/) job was not disabled and the original I-build is no longer available you can use the promoted RC2 build. - Change the DL_TYPE from S to R. - TAG will be set to R as well, for example `R4_27` - You can subscribe to [cross-project-issues](https://accounts.eclipse.org/mailing-list/cross-project-issues-dev) to get the notifications on Simrel releases.