From f2850855fcfcc4d0cc22b96036a04ad5742c9137 Mon Sep 17 00:00:00 2001 From: Hannes Wellmann Date: Mon, 7 Jul 2025 21:56:59 +0200 Subject: [PATCH 1/2] [RelEng] Unify and rename jobs for promoting and publishing builds --- JenkinsJobs/Releng/FOLDER.groovy | 20 ++++++------- ...e.jenkinsfile => promoteBuild.jenkinsfile} | 0 ...sfile => publishPromotedBuild.jenkinsfile} | 30 +++++++++---------- RELEASE.md | 14 ++++----- 4 files changed, 32 insertions(+), 32 deletions(-) rename JenkinsJobs/Releng/{renameAndPromote.jenkinsfile => promoteBuild.jenkinsfile} (100%) rename JenkinsJobs/Releng/{makeVisible.jenkinsfile => publishPromotedBuild.jenkinsfile} (66%) diff --git a/JenkinsJobs/Releng/FOLDER.groovy b/JenkinsJobs/Releng/FOLDER.groovy index a4c419384cb..1565eed8366 100644 --- a/JenkinsJobs/Releng/FOLDER.groovy +++ b/JenkinsJobs/Releng/FOLDER.groovy @@ -95,18 +95,18 @@ pipelineJob('Releng/prepareNextDevCycle'){ } } -pipelineJob('Releng/renameAndPromote'){ - displayName('Rename and Promote') +pipelineJob('Releng/promoteBuild'){ + displayName('Promote Build') description('''\ This job does the "stage 1" or first part of a promotion. -It renames the files for Equinox and Eclipse, creates an appropriate repo on 'downloads', rsync's everything to 'downloads', but leave everything "invisible" -- unless someone knows the exact URL. +It renames the files for Equinox and Eclipse, creates an appropriate repo on 'downloads', sync's everything to 'downloads', but leave everything "invisible" -- unless someone knows the exact URL. This allows two things. First, allows artifacts some time to "mirror" when that is needed. But also, allows the sites and repositories to be examined for correctness before making them visible to the world. The second (deferred) step that makes things visible works, in part, based on some output of this first step. Hence, they must "share a workspace". ''') parameters { stringParam('DROP_ID', null, '''\ -The name (or, build id) of the build to rename and promote. Typically would be a value such as I20160530-2000 or M20160912-1000. +The name (or, build id) of the build to promote. Typically would be a value such as 'I20250714-1800'. It must match the name of the build on the build machine. ''') stringParam('CHECKPOINT', null, 'M1, M3, RC1, RC2, RC3 etc (blank for final releases).') @@ -125,7 +125,7 @@ S is used for milestones and includes the milestone version. For example: S4_25_ scm { github('eclipse-platform/eclipse.platform.releng.aggregator', 'master') } - scriptPath('JenkinsJobs/Releng/renameAndPromote.jenkinsfile') + scriptPath('JenkinsJobs/Releng/promoteBuild.jenkinsfile') } } } @@ -155,11 +155,11 @@ GitHub issue to track tagging the release, for example: } } -pipelineJob('Releng/makeVisible'){ - displayName('Make Visible') +pipelineJob('Releng/publishPromotedBuild'){ + displayName('Publish Promoted Build') description('''\ -Make a 'release build', which was previously declared by running the 'Rename And Promote' job, visible. -The first part of a promotion -- the 'Rename And Promote' job -- puts the build at its final location, but keeps it hidden. +Make a 'release build', which was previously declared by running the 'Promote Build' job, visible. +The first part of a promotion -- the 'Promote Build' job -- puts the build at its final location, but keeps it hidden. Therefore, both jobs have to share a 'workspace', and the output of the first job must remain in place until its time to "make visible". ''') parameters { @@ -175,7 +175,7 @@ It must match the name of the build on the download server. scm { github('eclipse-platform/eclipse.platform.releng.aggregator', 'master') } - scriptPath('JenkinsJobs/Releng/makeVisible.jenkinsfile') + scriptPath('JenkinsJobs/Releng/publishPromotedBuild.jenkinsfile') } } } diff --git a/JenkinsJobs/Releng/renameAndPromote.jenkinsfile b/JenkinsJobs/Releng/promoteBuild.jenkinsfile similarity index 100% rename from JenkinsJobs/Releng/renameAndPromote.jenkinsfile rename to JenkinsJobs/Releng/promoteBuild.jenkinsfile diff --git a/JenkinsJobs/Releng/makeVisible.jenkinsfile b/JenkinsJobs/Releng/publishPromotedBuild.jenkinsfile similarity index 66% rename from JenkinsJobs/Releng/makeVisible.jenkinsfile rename to JenkinsJobs/Releng/publishPromotedBuild.jenkinsfile index 4618fcd0be8..84f6879eed7 100644 --- a/JenkinsJobs/Releng/makeVisible.jenkinsfile +++ b/JenkinsJobs/Releng/publishPromotedBuild.jenkinsfile @@ -15,10 +15,10 @@ pipeline { if (!params.releaseBuildID) { error "Required parameter 'releaseBuildID' is not defined." } - env.RELEASE_ID = params.releaseBuildID.trim() - def releaseIDMatcher = RELEASE_ID =~ /(?[SR])-(?\d+)\.(?\d+)(.\d+)?((M|RC)\d+[a-z]?)?-\d{12}/ + env.RELEASE_BUILD_ID = params.releaseBuildID.trim() + def releaseIDMatcher = env.RELEASE_BUILD_ID =~ /(?[SR])-(?\d+)\.(?\d+)(\.\d+)?((M|RC)\d+[a-z]?)?-\d{12}/ if (!releaseIDMatcher.matches()) { - error "releaseID: ${RELEASE_ID}, does not match the expected pattern." + error "releaseID: ${RELEASE_BUILD_ID}, does not match the expected pattern." } env.RELEASE_TYPE = releaseIDMatcher.group('type') env.RELEASE_VERSION_MAJOR = releaseIDMatcher.group('major') @@ -27,31 +27,31 @@ pipeline { } sh ''' echo 'Input parameters read successfully' - echo "RELEASE_ID='$RELEASE_ID'" + echo "RELEASE_BUILD_ID='$RELEASE_BUILD_ID'" echo "RELEASE_TYPE='$RELEASE_TYPE'" echo "RELEASE_VERSION_MAJOR='$RELEASE_VERSION_MAJOR'" echo "RELEASE_VERSION_MINOR='$RELEASE_VERSION_MINOR'" ''' } } - stage('Make Download page visible') { + stage('Make Download Page visible') { steps { sshagent(['projects-storage.eclipse.org-bot-ssh']) { sh '''#!/bin/bash -xe # Build machine locations (would very seldom change) DOWNLOAD_ROOT=${DOWNLOAD_ROOT:-/home/data/httpd/download.eclipse.org} ARCHIVE_ROOT=${ARCHIVE_ROOT:-/home/data/httpd/archive.eclipse.org} - ECLIPSE_MARKER="eclipse/downloads/drops4/${RELEASE_ID}/buildHidden" - EQUINOX_MARKER="equinox/drops/${RELEASE_ID}/buildHidden" - SSH_PREFIX="ssh genie.releng@projects-storage.eclipse.org" - - ${SSH_PREFIX} rm --force "${DOWNLOAD_ROOT}/${ECLIPSE_MARKER}" - ${SSH_PREFIX} rm --force "${DOWNLOAD_ROOT}/${EQUINOX_MARKER}" - + ROOT_LOCATIONS=("${DOWNLOAD_ROOT}") if [[ "${RELEASE_TYPE}" == 'R' ]]; then - ${SSH_PREFIX} rm --force "${ARCHIVE_ROOT}/${ECLIPSE_MARKER}" - ${SSH_PREFIX} rm --force "${ARCHIVE_ROOT}/${EQUINOX_MARKER}" + ROOT_LOCATIONS+=("${ARCHIVE_ROOT}") fi + MARKER_FILES=("eclipse/downloads/drops4/${RELEASE_BUILD_ID}/buildHidden" "equinox/drops/${RELEASE_BUILD_ID}/buildHidden") + + for rootLocation in "${ROOT_LOCATIONS[@]}"; do + for markerFile in "${MARKER_FILES[@]}"; do + ssh genie.releng@projects-storage.eclipse.org rm --force "${rootLocation}/${markerFile}" + done + done ''' } build job: 'Releng/updateIndex', wait: false @@ -64,7 +64,7 @@ pipeline { steps { build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [ string(name: 'repositoryPath', value: "eclipse/updates/${RELEASE_VERSION_MAJOR}.${RELEASE_VERSION_MINOR}"), - string(name: 'add', value: "${RELEASE_ID}") + string(name: 'add', value: "${RELEASE_BUILD_ID}") ] } } diff --git a/RELEASE.md b/RELEASE.md index bb5749eb456..ed606204e56 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -21,7 +21,7 @@ * All milestone releases are 'lightweight', meaning there is no announcement or signoff. No additional builds need to be run, just the daily I-build at 6PM EST. Thursdays build is promoted to simrel on friday (unless there are problems with Thursdays build, in which case promote Wednesdays) and the compiler is updated if necessary, - but the `Promote` and `Make Visible` jobs don't need to be run. + but the `Promote Build` and `Publish Promoted Build` jobs don't need to be run. - **Wednesday**: * Verify that EMF, ECF and Orbit contributions have been included (if applicable). * Final release candidate build runs at 6PM EST. @@ -35,7 +35,7 @@ * Just [1 line](https://www.eclipse.org/lists/platform-releng-dev/msg38086.html) asking for sign off on the GitHub issue created in the previous step. - **Friday**: * **Promote** the release candidate (if go). - * Run the [Rename and Promote](https://ci.eclipse.org/releng/job/Releng/job/renameAndPromote/) job in Jenkins + * Run the [Promote Build](https://ci.eclipse.org/releng/job/Releng/job/promoteBuild/) job in Jenkins - DROP_ID: Release candidate build ID (make sure there is no space before or after the ID). - CHECKPOINT: M1 etc (blank for final releases) - SIGNOFF_BUG: Needs to be updated to sign-off issue (numeric part only) @@ -43,7 +43,7 @@ - STREAM: 4.24.0 etc - DL_TYPE: S is used to promote I-builds. - TAG: Parameter should match stream version, i.e `S4_30_0_RC1` etc - - After the build find and open the mail template [artifact](https://ci.eclipse.org/releng/job/Releng/job/renameAndPromote/lastSuccessfulBuild/artifact/) and have it ready. + - After the build find and open the mail template [artifact](https://ci.eclipse.org/releng/job/Releng/job/promoteBuild/lastSuccessfulBuild/artifact/) and have it ready. - This should automatically run [tag Eclipse release](https://ci.eclipse.org/releng/job/Releng/job/tagEclipseRelease/) to tag the source code. * Contribute to SimRel - If you have not already set up SimRel you can do so using Auto Launch [here](https://www.eclipse.org/setups/installer/?url=https://git.eclipse.org/c/oomph/org.eclipse.oomph.git/plain/setups/interim/SimultaneousReleaseTrainConfiguration.setup&show=true) @@ -54,10 +54,10 @@ 4. Update the Location property to the "Specific repository for building against" in the mailtemplate.txt from promotion. 5. Commit Simrel updates to Gerrit - Message should use year-month format, i.e "Simrel updates for Eclipse and Equinox for 2022-06 M1" - * Run the [Make Visible](https://ci.eclipse.org/releng/job/Releng/job/makeVisible/) job in Releng jenkins to make the promoted build visible on the download page. - - `releaseBuildID`: the full id of the milestone, release-candidate or release build to make visible, e.g. `S-4.26M1-202209281800` or `R-4.36-202505281830` + * Run the [Publish Promoted Build](https://ci.eclipse.org/releng/job/Releng/job/publishPromotedBuild/) job in Releng jenkins to make the promoted build visible on the download page. + - `releaseBuildID`: the full id of the milestone, release-candidate or release build to publish, e.g. `S-4.26M1-202209281800` or `R-4.36-202505281830` * Send email that the M1 build is available - - Use the mail template from the promotion build [artifacts](https://ci.eclipse.org/releng/job/Releng/job/renameAndPromote/lastSuccessfulBuild/artifact/) in Jenkins to get the download urls. + - Use the mail template from the promotion build [artifacts](https://ci.eclipse.org/releng/job/Releng/job/promoteBuild/lastSuccessfulBuild/artifact/) in Jenkins to get the download urls. - Make sure to mention that the Master branch is now again open for development. * For **Milestone builds** return the I-builds to the normal schedule. * **After RC1** @@ -104,7 +104,7 @@ The actual steps to release **Friday** * #### **Promote to GA** - - After Simrel declares RC2 (usually the Friday before release) run the [Rename and Promote](https://ci.eclipse.org/releng/job/Releng/job/renameAndPromote/) job to promote RC2 (or RC2a). If the [daily cleanup for old builds](https://ci.eclipse.org/releng/job/Cleanup/job/dailyCleanOldBuilds/) job was not disabled and the original I-build is no longer available you can use the promoted RC2 build. + - After Simrel declares RC2 (usually the Friday before release) run the [Promote Build](https://ci.eclipse.org/releng/job/Releng/job/promoteBuild/) job to promote RC2 (or RC2a). If the [daily cleanup for old builds](https://ci.eclipse.org/releng/job/Cleanup/job/dailyCleanOldBuilds/) job was not disabled and the original I-build is no longer available you can use the promoted RC2 build. - Change the DL_TYPE from S to R. - TAG will be set to R as well, for example `R4_27` - You can subscribe to [cross-project-issues](https://accounts.eclipse.org/mailing-list/cross-project-issues-dev) to get the notifications on Simrel releases. From 3e9b69e3a58a72fdb430007e94e8db44f9d177ac Mon Sep 17 00:00:00 2001 From: Hannes Wellmann Date: Mon, 7 Jul 2025 22:04:52 +0200 Subject: [PATCH 2/2] [RelEng] Enhance parameter processing in build promotion job - Remove those parameters that can be derived from others. - Move creation of 'mailtemplate.txt' file to Jenkins pipeline. - Drop creation of unused 'checklist.txt' file. - Drop consideration of 'M'-DROP_IDs, which are not produced anymore. --- JenkinsJobs/Releng/FOLDER.groovy | 6 - JenkinsJobs/Releng/promoteBuild.jenkinsfile | 129 ++++++++++ RELEASE.md | 3 - cje-production/promotion/promoteSites.sh | 264 +------------------- 4 files changed, 132 insertions(+), 270 deletions(-) diff --git a/JenkinsJobs/Releng/FOLDER.groovy b/JenkinsJobs/Releng/FOLDER.groovy index 1565eed8366..c561214a10f 100644 --- a/JenkinsJobs/Releng/FOLDER.groovy +++ b/JenkinsJobs/Releng/FOLDER.groovy @@ -112,12 +112,6 @@ It must match the name of the build on the build machine. stringParam('CHECKPOINT', null, 'M1, M3, RC1, RC2, RC3 etc (blank for final releases).') stringParam('SIGNOFF_BUG', null, 'The issue that was used to "signoff" the checkpoint. If there are no unit test failures, this can be left blank. Otherwise a link is added to test page explaining that "failing unit tests have been investigated".') stringParam('TRAIN_NAME', null, 'The name of the release stream, typically yyyy-mm. For example: 2022-09') - stringParam('STREAM', null, 'Needs to be all three files of primary version for the release, such as 4.7.1 or 4.8.0.') - stringParam('DL_TYPE', null, "This is the build type we are promoting TO. I-builds promote to 'S' until 'R'.") - stringParam('TAG', null, ''' For passing to the tagEclipseRelease job. -R is used for release builds. For example: R4_25 -S is used for milestones and includes the milestone version. For example: S4_25_0_RC2 -''') } definition { cpsScm { diff --git a/JenkinsJobs/Releng/promoteBuild.jenkinsfile b/JenkinsJobs/Releng/promoteBuild.jenkinsfile index 3e87983f03e..49468c450f6 100644 --- a/JenkinsJobs/Releng/promoteBuild.jenkinsfile +++ b/JenkinsJobs/Releng/promoteBuild.jenkinsfile @@ -8,9 +8,122 @@ pipeline { agent { label 'basic' } + environment { + HIDE_SITE = 'true' + // Download Server locations (would very seldom change) + BUILD_ROOT = '/home/data/httpd/download.eclipse.org' + EP_ECLIPSE_ROOT = "${BUILD_ROOT}/eclipse" + EP_EQUINOX_ROOT = "${BUILD_ROOT}/equinox" + } stages { + stage('Process input') { + steps { + script { + env.DROP_ID = readParameter('DROP_ID') + if (!"${DROP_ID}") { + error("Parameter 'DROP_ID' is not specified") + } + env.CHECKPOINT = readParameter('CHECKPOINT') + env.SIGNOFF_BUG = readParameter('SIGNOFF_BUG') + env.TRAIN_NAME = readParameter('TRAIN_NAME') + if (!"${TRAIN_NAME}") { + error("Parameter 'TRAIN_NAME' is not specified") + } + def idMatcher = null + if ((idMatcher = env.DROP_ID =~ /(?I)(?\d{8})-(?