Enhance promotion job names and parameters

JenkinsJobs/Releng/FOLDER.groovy

@@ -95,37 +95,31 @@ pipelineJob('Releng/prepareNextDevCycle'){
 	}
 }
 
-pipelineJob('Releng/renameAndPromote'){
-	displayName('Rename and Promote')
+pipelineJob('Releng/promoteBuild'){
+	displayName('Promote Build')
 	description('''\
 This job does the "stage 1" or first part of a promotion.
-It renames the files for Equinox and Eclipse, creates an appropriate repo on 'downloads', rsync's everything to 'downloads', but leave everything "invisible" -- unless someone knows the exact URL.
+It renames the files for Equinox and Eclipse, creates an appropriate repo on 'downloads', sync's everything to 'downloads', but leave everything "invisible" -- unless someone knows the exact URL.
 This allows two things. First, allows artifacts some time to "mirror" when that is needed.
 But also, allows the sites and repositories to be examined for correctness before making them visible to the world.
 The second (deferred) step that makes things visible works, in part, based on some output of this first step. Hence, they must "share a workspace".
 ''')
 	parameters {
 		stringParam('DROP_ID', null, '''\
-The name (or, build id) of the build to rename and promote. Typically would be a value such as I20160530-2000 or M20160912-1000.
+The name (or, build id) of the build to promote. Typically would be a value such as 'I20250714-1800'.
 It must match the name of the build on the build machine.
 		''')
 		stringParam('CHECKPOINT', null, 'M1, M3, RC1, RC2, RC3 etc (blank for final releases).')
 		stringParam('SIGNOFF_BUG', null, 'The issue that was used to "signoff" the checkpoint. If there are no unit test failures, this can be left blank. Otherwise a link is added to test page explaining that "failing unit tests have been investigated".')
 		stringParam('TRAIN_NAME', null, 'The name of the release stream, typically yyyy-mm. For example: 2022-09')
-		stringParam('STREAM', null, 'Needs to be all three files of primary version for the release, such as 4.7.1 or 4.8.0.')
-		stringParam('DL_TYPE', null, "This is the build type we are promoting TO. I-builds promote to 'S' until 'R'.")
-		stringParam('TAG', null, ''' For passing to the tagEclipseRelease job.
-R is used for release builds. For example: R4_25
-S is used for milestones and includes the milestone version. For example: S4_25_0_RC2
-''')
 	}
 	definition {
 		cpsScm {
 			lightweight(true)
 			scm {
 				github('eclipse-platform/eclipse.platform.releng.aggregator', 'master')
 			}
-			scriptPath('JenkinsJobs/Releng/renameAndPromote.jenkinsfile')
+			scriptPath('JenkinsJobs/Releng/promoteBuild.jenkinsfile')
 		}
 	}
 }
@@ -155,11 +149,11 @@ GitHub issue to track tagging the release, for example:
 	}
 }
 
-pipelineJob('Releng/makeVisible'){
-	displayName('Make Visible')
+pipelineJob('Releng/publishPromotedBuild'){
+	displayName('Publish Promoted Build')
 	description('''\
-Make a 'release build', which was previously declared by running the 'Rename And Promote' job, visible.
-The first part of a promotion -- the 'Rename And Promote' job -- puts the build at its final location, but keeps it hidden.
+Make a 'release build', which was previously declared by running the 'Promote Build' job, visible.
+The first part of a promotion -- the 'Promote Build' job -- puts the build at its final location, but keeps it hidden.
 Therefore, both jobs have to share a 'workspace', and the output of the first job must remain in place until its time to "make visible".
 ''')
 	parameters {
@@ -175,7 +169,7 @@ It must match the name of the build on the download server.
 			scm {
 				github('eclipse-platform/eclipse.platform.releng.aggregator', 'master')
 			}
-			scriptPath('JenkinsJobs/Releng/makeVisible.jenkinsfile')
+			scriptPath('JenkinsJobs/Releng/publishPromotedBuild.jenkinsfile')
 		}
 	}
 }

JenkinsJobs/Releng/promoteBuild.jenkinsfile

@@ -0,0 +1,163 @@
+pipeline {
+	options {
+		skipDefaultCheckout()
+		timestamps()
+		timeout(time: 120, unit: 'MINUTES')
+		buildDiscarder(logRotator(numToKeepStr:'5'))
+	}
+	agent {
+		label 'basic'
+	}
+	environment {
+		HIDE_SITE = 'true'
+		// Download Server locations (would very seldom change)
+		BUILD_ROOT = '/home/data/httpd/download.eclipse.org'
+		EP_ECLIPSE_ROOT = "${BUILD_ROOT}/eclipse"
+		EP_EQUINOX_ROOT = "${BUILD_ROOT}/equinox"
+	}
+	stages {
+		stage('Process input') {
+			steps {
+				script {
+					env.DROP_ID = readParameter('DROP_ID')
+					if (!"${DROP_ID}") {
+						error("Parameter 'DROP_ID' is not specified")
+					}
+					env.CHECKPOINT = readParameter('CHECKPOINT')
+					env.SIGNOFF_BUG = readParameter('SIGNOFF_BUG')
+					env.TRAIN_NAME = readParameter('TRAIN_NAME')
+					if (!"${TRAIN_NAME}") {
+						error("Parameter 'TRAIN_NAME' is not specified")
+					}
+					def idMatcher = null
+					if ((idMatcher = env.DROP_ID =~ /(?<type>I)(?<date>\d{8})-(?<time>\d{4})/).matches()) {
+						assignEnvVariable('BUILD_LABEL', "${DROP_ID}")
+					} else if ((idMatcher = env.DROP_ID =~ /(?<type>S)-(?<major>\d+)\.(?<minor>\d+)(?<service>\.\d+)?((M|RC)\d+[a-z]?)?-(?<date>\d{8})(?<time>\d{4})/).matches()) {
+						assignEnvVariable('BUILD_LABEL', idMatcher.group('major') + '.' + idMatcher.group('minor') + (idMatcher.group('service') ?: ''))
+						if ("${CHECKPOINT}") {
+							error "Stable build DROP_ID=${DROP_ID} may only be promoted to release CHECKPOINT, which therefore must be empty: ${CHECKPOINT}"
+						}
+					} else {
+						error "DROP_ID, ${DROP_ID}, did not match any expected pattern."
+					}
+					assignEnvVariable('BUILD_TYPE', idMatcher.group('type'))
+					assignEnvVariable('REPO_BUILD_TYPE', 'I')
+					assignEnvVariable('BUILD_TIMESTAMP', idMatcher.group('date') + idMatcher.group('time'))
+					assignEnvVariable('REPO_ID', "I${idMatcher.group('date')}-${idMatcher.group('time')}")
+					idMatcher = null // release matcher as it's not serializable
+
+					assignEnvVariable('BUILD_LABEL_EQ', "${BUILD_LABEL}")
+					assignEnvVariable('DROP_ID_EQ', "${DROP_ID}")
+
+					sh 'curl -o buildproperties.shsource --fail https://download.eclipse.org/eclipse/downloads/drops4/${DROP_ID}/buildproperties.shsource'
+					def STREAM = sh(returnStdout: true, script: 'source ./buildproperties.shsource && echo ${STREAM}').trim()
+					def versionMatcher = STREAM =~ /(?<major>\d+)\.(?<minor>\d+).(?<service>\d+)/
+					if (!versionMatcher.matches()) {
+						error "STREAM must contain major, minor, and service versions, such as '4.37.0', but found: ${STREAM}"
+					}
+					assignEnvVariable('BUILD_MAJOR', versionMatcher.group('major'))
+					assignEnvVariable('BUILD_MINOR', versionMatcher.group('minor'))
+					assignEnvVariable('BUILD_SERVICE', versionMatcher.group('service'))
+					versionMatcher = null // release matcher as it's not serializable
+
+					if ("${CHECKPOINT}" ==~ /M\d+([a-z])?/ || "${CHECKPOINT}" ==~ /RC\d+([a-z])?/) { // milestone or RC promotion
+						assignEnvVariable('DL_TYPE', 'S')
+						// REPO_SITE_SEGMENT variale not used in this case
+					} else if(!"${CHECKPOINT}") { // release promotion
+						assignEnvVariable('DL_TYPE', 'R')
+						assignEnvVariable('REPO_SITE_SEGMENT', "${BUILD_MAJOR}.${BUILD_MINOR}")
+					} else {
+						error "CHECKPOINT, ${CHECKPOINT}, did not match any expected pattern."
+					}
+					assignEnvVariable('NEWS_ID', "${BUILD_MAJOR}.${BUILD_MINOR}")
+
+					assignEnvVariable('DL_LABEL', "${BUILD_SERVICE}" == '0' // For initial releases, do not include service in label
+						? "${BUILD_MAJOR}.${BUILD_MINOR}${CHECKPOINT}"
+						: "${BUILD_MAJOR}.${BUILD_MINOR}.${BUILD_SERVICE}${CHECKPOINT}"
+					)
+					assignEnvVariable('DL_LABEL_EQ', "${DL_LABEL}")
+
+					// This is DL_DROP_ID for Eclipse. The one for equinox has DL_LABEL_EQ in middle.
+					assignEnvVariable('DL_DROP_ID', "${DL_TYPE}-${DL_LABEL}-${BUILD_TIMESTAMP}")
+					assignEnvVariable('DL_DROP_ID_EQ', "${DL_TYPE}-${DL_LABEL_EQ}-${BUILD_TIMESTAMP}")
+
+					if (!env.SIGNOFF_BUG) {
+						echo '''\
+							[WARNING] SIGNOFF_BUG was not defined. That is valid if no Unit Tests failures but otherwise should be defined.
+							Can be added by hand to buildproperties.php in drop site, if in fact there were errors, and simply forgot to specify.
+						'''.stripIndent()
+					}
+
+					def serviceVersionSegment = (env.CHECKPOINT || env.BUILD_SERVICE != '0') ? ('_' + env.BUILD_SERVICE) : ''
+					assignEnvVariable('TAG', "${DL_TYPE}${BUILD_MAJOR}_${BUILD_MINOR}${serviceVersionSegment}${env.CHECKPOINT ? ('_' + env.CHECKPOINT) : ''}")
+				}
+			}
+		}
+		stage('Rename and Promote') {
+			environment {
+				BUILDMACHINE_BASE_DL = "${EP_ECLIPSE_ROOT}/downloads/drops4"
+				BUILDMACHINE_BASE_EQ = "${EP_EQUINOX_ROOT}/drops"
+				BUILD_REPO_ORIGINAL = "${BUILD_MAJOR}.${BUILD_MINOR}-${REPO_BUILD_TYPE}-builds"
+				BUILDMACHINE_BASE_SITE = "${EP_ECLIPSE_ROOT}/updates/${BUILD_REPO_ORIGINAL}"
+				// Eclipse and Equinox drop Site (final segment)
+				ECLIPSE_DL_DROP_DIR_SEGMENT = "${DL_TYPE}-${DL_LABEL}-${BUILD_TIMESTAMP}"
+				EQUINOX_DL_DROP_DIR_SEGMENT = "${DL_TYPE}-${DL_LABEL_EQ}-${BUILD_TIMESTAMP}"
+			}
+			steps {
+				writeFile(file: "${WORKSPACE}/stage2output${TRAIN_NAME}${CHECKPOINT}/mailtemplate.txt", text: """\
+					We are pleased to announce that ${TRAIN_NAME} ${CHECKPOINT} is available for download and updates.
+
+						Eclipse downloads:
+						https://download.eclipse.org/eclipse/downloads/drops4/${ECLIPSE_DL_DROP_DIR_SEGMENT}/
+
+						New and Noteworthy:
+						https://www.eclipse.org/eclipse/news/${NEWS_ID}/
+
+						Update existing (non-production) installs:
+						https://download.eclipse.org/eclipse/updates/${DL_TYPE == 'R' ? REPO_SITE_SEGMENT : BUILD_REPO_ORIGINAL}/
+
+						Specific repository good for building against:
+						https://download.eclipse.org/eclipse/updates/${DL_TYPE == 'R' ? (REPO_SITE_SEGMENT + '/' + ECLIPSE_DL_DROP_DIR_SEGMENT) : (BUILD_REPO_ORIGINAL + '/' + DROP_ID)}/
+
+						Equinox specific downloads:
+						https://download.eclipse.org/equinox/drops/${EQUINOX_DL_DROP_DIR_SEGMENT}/
+
+					Thank you to everyone who made this checkpoint possible.
+				""".stripIndent())
+				sshagent(['projects-storage.eclipse.org-bot-ssh']) {
+					sh '''#!/bin/bash -x
+						curl -o promoteSites.sh https://download.eclipse.org/eclipse/relengScripts/cje-production/promotion/promoteSites.sh
+						chmod +x promoteSites.sh
+						./promoteSites.sh
+					'''
+				}
+				build job: 'Releng/tagEclipseRelease', wait: true, propagate: true, parameters: [
+					string(name: 'tag', value: "${TAG}"),
+					string(name: 'buildID', value: "${DROP_ID}"),
+					string(name: 'annotation', value: "${ params.SIGNOFF_BUG ? 'https://github.com/eclipse-platform/eclipse.platform.releng.aggregator/issues/' + params.SIGNOFF_BUG : '' }")
+				]
+			}
+		}
+	}
+	post {
+		always {
+			archiveArtifacts '**/stage2output*/**'
+		}
+	}
+}
+
+@NonCPS
+def assignEnvVariable(String name, String value) {
+	env."${name}" = value
+	println("${name}=${value}")
+}
+
+@NonCPS
+def readParameter(String name) {
+	//TODO: let jenkins trim the parameters
+	def value = (params[name] ?: '').trim()
+	if (value) {
+		println("${name}: ${value}")
+	}
+	return value
+}

JenkinsJobs/Releng/makeVisible.jenkinsfile → JenkinsJobs/Releng/publishPromotedBuild.jenkinsfile

@@ -15,10 +15,10 @@ pipeline {
 					if (!params.releaseBuildID) {
 						error "Required parameter 'releaseBuildID' is not defined."
 					}
-					env.RELEASE_ID = params.releaseBuildID.trim()
-					def releaseIDMatcher = RELEASE_ID =~ /(?<type>[SR])-(?<major>\d+)\.(?<minor>\d+)(.\d+)?((M|RC)\d+[a-z]?)?-\d{12}/
+					env.RELEASE_BUILD_ID = params.releaseBuildID.trim()
+					def releaseIDMatcher = env.RELEASE_BUILD_ID =~ /(?<type>[SR])-(?<major>\d+)\.(?<minor>\d+)(\.\d+)?((M|RC)\d+[a-z]?)?-\d{12}/
 					if (!releaseIDMatcher.matches()) {
-						error "releaseID: ${RELEASE_ID}, does not match the expected pattern."
+						error "releaseID: ${RELEASE_BUILD_ID}, does not match the expected pattern."
 					}
 					env.RELEASE_TYPE = releaseIDMatcher.group('type')
 					env.RELEASE_VERSION_MAJOR = releaseIDMatcher.group('major')
@@ -27,31 +27,31 @@ pipeline {
 				}
 				sh '''
 					echo 'Input parameters read successfully'
-					echo "RELEASE_ID='$RELEASE_ID'"
+					echo "RELEASE_BUILD_ID='$RELEASE_BUILD_ID'"
 					echo "RELEASE_TYPE='$RELEASE_TYPE'"
 					echo "RELEASE_VERSION_MAJOR='$RELEASE_VERSION_MAJOR'"
 					echo "RELEASE_VERSION_MINOR='$RELEASE_VERSION_MINOR'"
 				'''
 			}
 		}
-		stage('Make Download page visible') {
+		stage('Make Download Page visible') {
 			steps {
 				sshagent(['projects-storage.eclipse.org-bot-ssh']) {
 					sh '''#!/bin/bash -xe
 						# Build machine locations (would very seldom change)
 						DOWNLOAD_ROOT=${DOWNLOAD_ROOT:-/home/data/httpd/download.eclipse.org}
 						ARCHIVE_ROOT=${ARCHIVE_ROOT:-/home/data/httpd/archive.eclipse.org} 
-						ECLIPSE_MARKER="eclipse/downloads/drops4/${RELEASE_ID}/buildHidden"
-						EQUINOX_MARKER="equinox/drops/${RELEASE_ID}/buildHidden"
-						SSH_PREFIX="ssh [email protected]"
-
-						${SSH_PREFIX} rm --force "${DOWNLOAD_ROOT}/${ECLIPSE_MARKER}"
-						${SSH_PREFIX} rm --force "${DOWNLOAD_ROOT}/${EQUINOX_MARKER}"
-
+						ROOT_LOCATIONS=("${DOWNLOAD_ROOT}")
 						if [[ "${RELEASE_TYPE}" == 'R' ]]; then
-							${SSH_PREFIX} rm --force "${ARCHIVE_ROOT}/${ECLIPSE_MARKER}"
-							${SSH_PREFIX} rm --force "${ARCHIVE_ROOT}/${EQUINOX_MARKER}"
+							ROOT_LOCATIONS+=("${ARCHIVE_ROOT}")
 						fi
+						MARKER_FILES=("eclipse/downloads/drops4/${RELEASE_BUILD_ID}/buildHidden" "equinox/drops/${RELEASE_BUILD_ID}/buildHidden")
+
+						for rootLocation in "${ROOT_LOCATIONS[@]}"; do
+							for markerFile in "${MARKER_FILES[@]}"; do
+								ssh [email protected] rm --force "${rootLocation}/${markerFile}"
+							done
+						done
 					'''
 				}
 				build job: 'Releng/updateIndex', wait: false
@@ -64,7 +64,7 @@ pipeline {
 			steps {
 				build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
 					string(name: 'repositoryPath', value: "eclipse/updates/${RELEASE_VERSION_MAJOR}.${RELEASE_VERSION_MINOR}"),
-					string(name: 'add', value: "${RELEASE_ID}")
+					string(name: 'add', value: "${RELEASE_BUILD_ID}")
 				]
 			}
 		}