[RelEng] Miscellaneous fixes and clean-ups in promotion/preparation

- Fix currently not working calls to gitCommitAllExcludingSubmodules(),
because methods on objects may only be called in script blocks or
methods.

- Enable running the preparation of a new release with an I-build as
PREVIOUS_RELEASE_CANDIDATE_ID.

- Fix check of response status in githubAPI utility script. The status
was compared as String with an integer and not parsed, leading to always
false comparisons.

- Skip checkout of maintenance branch for non release promotions

- Remove unnecessary arguments of Maven invocations and other minor
clean-ups

Author: HannesWell

JenkinsJobs/Releng/prepareNextDevCycle.jenkinsfile

@@ -29,18 +29,28 @@ pipeline {
 					nextVersionMatcher = null // release matcher as it's not serializable
 
 					env.PREVIOUS_RELEASE_CANDIDATE_ID = readParameter('PREVIOUS_RELEASE_CANDIDATE_ID')
-					def previousIdMatcher = env.PREVIOUS_RELEASE_CANDIDATE_ID =~ /(?<type>[SR])-(?<major>\d+)\.(?<minor>\d+)(\.(?<service>\d+))?(?<checkpoint>(M|RC)\d+[a-z]?)?-(?<date>\d{8})(?<time>\d{4})/
-					if (!previousIdMatcher.matches()) {
+					def previousIdMatcher = null
+					if ((previousIdMatcher = env.PREVIOUS_RELEASE_CANDIDATE_ID =~ /(?<type>[SR])-(?<major>\d+)\.(?<minor>\d+)(\.(?<service>\d+))?(?<checkpoint>(M|RC)\d+[a-z]?)?-(?<date>\d{8})(?<time>\d{4})/).matches()) {
+						def checkpoint = previousIdMatcher.group('checkpoint')
+						assignEnvVariable('PREVIOUS_RELEASE_VERSION_MAJOR', previousIdMatcher.group('major'))
+						assignEnvVariable('PREVIOUS_RELEASE_VERSION_MINOR', previousIdMatcher.group('minor'))
+						assignEnvVariable('PREVIOUS_RELEASE_CANDIDATE_TAG', "${PREVIOUS_RELEASE_VERSION_MAJOR}.${PREVIOUS_RELEASE_VERSION_MINOR}${checkpoint}")
+						def previousServiceVersion = previousIdMatcher.group('service') ?: '0'
+						assignEnvVariable('PREVIOUS_RELEASE_CANDIDATE_GIT_TAG', "${previousIdMatcher.group('type')}${PREVIOUS_RELEASE_VERSION_MAJOR}_${PREVIOUS_RELEASE_VERSION_MINOR}${(checkpoint || previousServiceVersion != '0') ? ('_' + previousServiceVersion) : ''}${checkpoint ? ('_' + checkpoint) : ''}")
+						
+					} else if ((previousIdMatcher = env.PREVIOUS_RELEASE_CANDIDATE_ID =~ /I(?<date>\d{8})-(?<time>\d{4})/).matches()) {
+						def buildPropertiesTxt = sh(script: "curl --fail https://download.eclipse.org/eclipse/downloads/drops4/${PREVIOUS_RELEASE_CANDIDATE_ID}/buildproperties.txt", returnStdout: true)
+						def buildProperties = readProperties(text: buildPropertiesTxt)
+						assignEnvVariable('PREVIOUS_RELEASE_VERSION_MAJOR', buildProperties.STREAMMajor.replace('"','')) // Remove surrounding quotes
+						assignEnvVariable('PREVIOUS_RELEASE_VERSION_MINOR', buildProperties.STREAMMinor.replace('"','')) // Remove surrounding quotes
+						assignEnvVariable('PREVIOUS_RELEASE_CANDIDATE_TAG', "${PREVIOUS_RELEASE_CANDIDATE_ID}")
+						assignEnvVariable('PREVIOUS_RELEASE_CANDIDATE_GIT_TAG', "${PREVIOUS_RELEASE_CANDIDATE_ID}")
+						
+					} else {
 						error "Unexpected format for PREVIOUS_RELEASE_CANDIDATE_ID: ${PREVIOUS_RELEASE_CANDIDATE_ID}"
 					}
-					def checkpoint = previousIdMatcher.group('checkpoint')
-					def previousReleaseVersionService = previousIdMatcher.group('service') ?: '0'
-					assignEnvVariable('PREVIOUS_RELEASE_VERSION_MAJOR', previousIdMatcher.group('major'))
-					assignEnvVariable('PREVIOUS_RELEASE_VERSION_MINOR', previousIdMatcher.group('minor'))
 					assignEnvVariable('PREVIOUS_RELEASE_VERSION', "${PREVIOUS_RELEASE_VERSION_MAJOR}.${PREVIOUS_RELEASE_VERSION_MINOR}")
-					assignEnvVariable('PREVIOUS_RELEASE_CANDIDATE_TAG', "${PREVIOUS_RELEASE_VERSION}${checkpoint}")
 					assignEnvVariable('PREVIOUS_RELEASE_CANDIDATE_I_BUILD', "I${previousIdMatcher.group('date')}-${previousIdMatcher.group('time')}")
-					assignEnvVariable('PREVIOUS_RELEASE_CANDIDATE_GIT_TAG', "${previousIdMatcher.group('type')}${PREVIOUS_RELEASE_VERSION_MAJOR}_${PREVIOUS_RELEASE_VERSION_MINOR}${(checkpoint || previousReleaseVersionService != '0') ? ('_' + previousReleaseVersionService) : ''}${checkpoint ? ('_' + checkpoint) : ''}")
 					previousIdMatcher = null // release matcher as it's not serializable
 
 					//TODO: Read the dates from the calender instead of provide a structured document somewhere?
@@ -100,14 +110,11 @@ pipeline {
 			}
 		}
 		stage('Update Maven Version') {
-			environment {
-				MAVEN_ARGS = '-U -B -ntp'
-			}
 			steps {
 				sh '''
 					mvn org.eclipse.tycho:tycho-versions-plugin:set-version \
 						-DnewVersion=${NEXT_RELEASE_VERSION}.0-SNAPSHOT
-					mvn -f eclipse-platform-parent/pom.xml --non-recursive org.eclipse.tycho:tycho-versions-plugin:set-property \
+					mvn -f eclipse-platform-parent/pom.xml tycho-versions:set-property \
 						-Dproperties=releaseVersion,releaseYear,releaseMonth \
 						-DnewReleaseVersion=${NEXT_RELEASE_VERSION} \
 						-DnewReleaseYear=${NEXT_RELEASE_YEAR} \
@@ -155,13 +162,13 @@ pipeline {
 					"(?<prefix># Schedule:.*\\R)(?s).*(?<suffix>\\R'''\\))" : "\${prefix}${I_BUILD_SCHEDULE}\${suffix}",
 				])
 
-				utilities.commitAllChangesExcludingSubmodules("Update versions to ${NEXT_RELEASE_VERSION} in build scripts")
+				gitCommitAllExcludingSubmodules("Update versions to ${NEXT_RELEASE_VERSION} in build scripts")
 			}
 		}
 		stage('Move previous version to current RC') {
 			steps {
 				sh '''
-					mvn -f eclipse-platform-parent/pom.xml --non-recursive org.eclipse.tycho:tycho-versions-plugin:set-property \
+					mvn -f eclipse-platform-parent/pom.xml tycho-versions:set-property \
 						-Dproperties=previous-release.baseline \
 						"-DnewPrevious-release.baseline=https://download.eclipse.org/eclipse/updates/${PREVIOUS_RELEASE_VERSION}-I-builds/${PREVIOUS_RELEASE_CANDIDATE_I_BUILD}/"
 				'''
@@ -183,7 +190,8 @@ pipeline {
 					'previousReleaseVersion=.*' : "previousReleaseVersion=${PREVIOUS_RELEASE_CANDIDATE_TAG}",
 					'previousReleaseVersionRepo=.*' : "previousReleaseVersionRepo=${PREVIOUS_RELEASE_VERSION}-I-builds",
 				])
-				utilities.commitAllChangesExcludingSubmodules("Move previous version to ${PREVIOUS_RELEASE_CANDIDATE_TAG} in build scripts")
+				
+				gitCommitAllExcludingSubmodules("Move previous version to ${PREVIOUS_RELEASE_CANDIDATE_TAG} in build scripts")
 			}
 		}
 		stage('Clear Qualifier-update files') {
@@ -243,7 +251,7 @@ pipeline {
 				])
 				sh "mv cje-production/streams/repositories_master.txt cje-production/streams/repositories_${MAINTENANCE_BRANCH}.txt"
 
-				utilities.commitAllChangesExcludingSubmodules("Move ${PREVIOUS_RELEASE_VERSION}-I builds to ${MAINTENANCE_BRANCH} branch")
+				gitCommitAllExcludingSubmodules("Move ${PREVIOUS_RELEASE_VERSION}-I builds to ${MAINTENANCE_BRANCH} branch")
 
 				// Switch back to master for subsequent parts of this pipeline
 				sh 'git checkout master'
@@ -274,7 +282,7 @@ pipeline {
 						--fixed-strings "${PREVIOUS_RELEASE_VERSION}")
 					# The eclipse-platform-parent/pom.xml contains the previous version in the baseline repository variable
 					if [[ -z "${matchingFiles}" ]] || [[ "${matchingFiles}" == 'eclipse-platform-parent/pom.xml' ]]; then
-						echo "No unexpected references to previous version ${PREVIOUS_RELEASE_VERSION} found:"
+						echo "No unexpected references to previous version ${PREVIOUS_RELEASE_VERSION} found."
 						exit 0
 					else
 						echo "References to previous version ${PREVIOUS_RELEASE_VERSION} found:"
@@ -373,7 +381,7 @@ pipeline {
 						- Updating the release version to `${NEXT_RELEASE_VERSION}` across build scripts
 						- Updating the previous release version to the current Release-Candidate: `${PREVIOUS_RELEASE_CANDIDATE_ID}`
 						""".stripIndent(), prBranch)
-						
+					
 					def submodulePaths = sh(script: "git submodule --quiet foreach 'echo \$sm_path'", returnStdout: true).trim().split('\\s')
 					for (submodulePath in submodulePaths) {
 						def diff = sh(script:"cd ${submodulePath} && git diff master origin/master --shortstat", returnStdout: true).trim()
@@ -469,3 +477,7 @@ def replaceInFile(String filePath, Map<String,String> replacements) {
 def replaceAllInFile(String filePath, Map<String,String> replacements) {
 	utilities.replaceAllInFile(filePath, replacements)
 }
+
+def gitCommitAllExcludingSubmodules(String commitMessage) {
+	utilities.gitCommitAllExcludingSubmodules(commitMessage)
+}

JenkinsJobs/Releng/promoteBuild.jenkinsfile

@@ -9,7 +9,6 @@ pipeline {
 		label 'basic'
 	}
 	environment {
-		HIDE_SITE = 'true'
 		// Download Server locations (would very seldom change)
 		EP_ROOT = '/home/data/httpd/download.eclipse.org'
 	}
@@ -42,16 +41,11 @@ pipeline {
 					assignEnvVariable('REPO_ID', "I${idMatcher.group('date')}-${idMatcher.group('time')}")
 					idMatcher = null // release matcher as it's not serializable
 
-					sh 'curl -o buildproperties.shsource --fail https://download.eclipse.org/eclipse/downloads/drops4/${DROP_ID}/buildproperties.shsource'
-					def STREAM = sh(returnStdout: true, script: 'source ./buildproperties.shsource && echo ${STREAM}').trim()
-					def versionMatcher = STREAM =~ /(?<major>\d+)\.(?<minor>\d+).(?<service>\d+)/
-					if (!versionMatcher.matches()) {
-						error "STREAM must contain major, minor, and service versions, such as '4.37.0', but found: ${STREAM}"
-					}
-					assignEnvVariable('BUILD_MAJOR', versionMatcher.group('major'))
-					assignEnvVariable('BUILD_MINOR', versionMatcher.group('minor'))
-					assignEnvVariable('BUILD_SERVICE', versionMatcher.group('service'))
-					versionMatcher = null // release matcher as it's not serializable
+					def buildPropertiesTxt = sh(script: "curl --fail https://download.eclipse.org/eclipse/downloads/drops4/${DROP_ID}/buildproperties.txt", returnStdout: true)
+					def buildProperties = readProperties(text: buildPropertiesTxt)
+					assignEnvVariable('BUILD_MAJOR', buildProperties.STREAMMajor.replace('"','')) // Remove surrounding quotes
+					assignEnvVariable('BUILD_MINOR', buildProperties.STREAMMinor.replace('"','')) // Remove surrounding quotes
+					assignEnvVariable('BUILD_SERVICE', buildProperties.STREAMService.replace('"','')) // Remove surrounding quotes
 
 					if ("${CHECKPOINT}" ==~ /M\d+([a-z])?/ || "${CHECKPOINT}" ==~ /RC\d+([a-z])?/) { // milestone or RC promotion
 						assignEnvVariable('DL_TYPE', 'S')
@@ -90,9 +84,12 @@ pipeline {
 					}
 					sh '''#!/bin/bash -xe
 						git branch --force master HEAD
-						git fetch origin "refs/heads/${MAINTENANCE_BRANCH}"
-						git reflog show master
-						git reflog show "origin/${MAINTENANCE_BRANCH}"
+						if [[ "${DL_TYPE}" == 'R' ]]; then
+							git fetch origin "refs/heads/${MAINTENANCE_BRANCH}"
+							
+							git reflog show master
+							git reflog show "origin/${MAINTENANCE_BRANCH}"
+						fi
 
 						git fetch origin tag "${REPO_ID}"
 						git checkout "${REPO_ID}"
@@ -196,7 +193,7 @@ pipeline {
 					script { // Update the master branch
 						sh 'git checkout master'
 						sh '''
-							mvn -f eclipse-platform-parent/pom.xml org.eclipse.tycho:tycho-versions-plugin:set-property \
+							mvn -f eclipse-platform-parent/pom.xml tycho-versions:set-property \
 								-Dproperties=previous-release.baseline \
 								-DnewPrevious-release.baseline="${RELEASE_P2_REPOSITORY}"
 						'''
@@ -225,15 +222,15 @@ pipeline {
 							'previousReleaseVersionRepo=.*' : "previousReleaseVersionRepo=${BUILD_MAJOR}.${BUILD_MINOR}",
 						])
 
-						utilities.commitAllChangesExcludingSubmodules("Update previous release version to ${BUILD_MAJOR}.${BUILD_MINOR} GA across build scripts")
+						utilities.gitCommitAllExcludingSubmodules("Update previous release version to ${BUILD_MAJOR}.${BUILD_MINOR} GA across build scripts")
 					}
 					script { // Update the maintenance branch
 						sh 'git checkout -b updateMaintenance "origin/${MAINTENANCE_BRANCH}"'
-						def ecjManifest = sh(script: "curl https://raw.githubusercontent.com/eclipse-jdt/eclipse.jdt.core/refs/tags/${REPO_ID}/org.eclipse.jdt.core.compiler.batch/META-INF/MANIFEST.MF", returnStdout: true).trim()
+						def ecjManifest = sh(script: "curl --fail https://raw.githubusercontent.com/eclipse-jdt/eclipse.jdt.core/refs/tags/${REPO_ID}/org.eclipse.jdt.core.compiler.batch/META-INF/MANIFEST.MF", returnStdout: true).trim()
 						def bundleVersion = readManifest(text: ecjManifest).main['Bundle-Version']
 						def ecjVersion = bundleVersion.substring(0, bundleVersion.indexOf('.qualifier'))
 						sh """
-							mvn -f eclipse-platform-parent/pom.xml org.eclipse.tycho:tycho-versions-plugin:set-property \
+							mvn -f eclipse-platform-parent/pom.xml tycho-versions:set-property \
 								-Dproperties=eclipse-sdk-repo,previous-release.baseline,cbi-ecj-version \
 								-DnewEclipse-sdk-repo="${RELEASE_P2_REPOSITORY}" \
 								-DnewPrevious-release.baseline="${RELEASE_P2_REPOSITORY}" \
@@ -243,7 +240,7 @@ pipeline {
 							'ECLIPSE_RUN_REPO="https://download.eclipse.org/eclipse/updates/.*"' : "ECLIPSE_RUN_REPO=\"${RELEASE_P2_REPOSITORY}\"",
 						])
 
-						utilities.commitAllChangesExcludingSubmodules("Update ${MAINTENANCE_BRANCH} branch with release version for ${BUILD_MAJOR}_${BUILD_MINOR}+ changes")
+						utilities.gitCommitAllExcludingSubmodules("Update ${MAINTENANCE_BRANCH} branch with release version for ${BUILD_MAJOR}_${BUILD_MINOR}+ changes")
 					}
 					// Switch back to master for subsequent parts of this pipeline
 					sh 'git checkout master'
@@ -330,10 +327,9 @@ def renameBuildDrop(String baseDropPath, String oldDropID, String oldBuildLabel,
 
 		# Copy drop-directory to new location
 		ssh [email protected] mkdir -p '${targetPath}'
-		if [[ "${HIDE_SITE}" == "true" ]]; then
-			# Create this marker first to ensure the page is never without
-			ssh [email protected] touch '${targetPath}/buildHidden'
-		fi
+		# Create this marker first to ensure the page is never without
+		ssh [email protected] touch '${targetPath}/buildHidden'
+		
 		ssh [email protected] cp -r '${sourcePath}/.' '${targetPath}'
 
 		# Rename files

JenkinsJobs/shared/githubAPI.groovy

@@ -9,7 +9,7 @@ def setDryRun(boolean isDryRun) {
 /** Returns a list of all repositories in the specified organization.*/
 def listReposOfOrganization(String orga) {
 	def response = queryGithubAPI('', "orgs/${orga}/repos", null)
-	if (!(response instanceof List) && (response.errors || (response.status && response.status != 201))) {
+	if (!(response instanceof List) && isFailed(response, 201)) {
 		error "Response contains errors:\n${response}"
 	}
 	return response
@@ -23,7 +23,7 @@ def createMilestone(String orga, String repo, String msTitle, String msDescripti
 	echo "In ${orga}/${repo} create milestone: ${msTitle} due on ${msDueDay}"
 	def params = [title: msTitle, description: msDescription, due_on: "${msDueDay}T23:59:59Z"]
 	def response = queryGithubAPI('-X POST', "repos/${orga}/${repo}/milestones", params)
-	if (response?.errors || (response?.status && response.status != 201)) {
+	if (isFailed(response, 201)) {
 		if (response.errors && response.errors[0]?.code == 'already_exists') {
 			echo 'Milestone already exists and is not modified'
 			// TODO: update milestone in this case: https://docs.github.com/en/rest/issues/milestones?apiVersion=2022-11-28#update-a-milestone
@@ -41,7 +41,7 @@ def createPullRequest(String orgaSlashRepo, String prTitle, String prBody, Strin
 	echo "In ${orgaSlashRepo} create PR: '${prTitle}' on branch ${headBranch}"
 	def params = [title: prTitle, body: prBody, head: headBranch, base: baseBranch]
 	def response = queryGithubAPI('-X POST',"repos/${orgaSlashRepo}/pulls", params)
-	if (response?.errors || (response?.status && response.status != 201)) {
+	if (isFailed(response, 201)) {
 		if (skipExistingPR && response.errors[0]?.message?.contains('pull request already exists')) {
 			echo "Ignoring failure to create pull request: ${response.errors[0].message}"
 			return null; // PR already exists and the caller asked to ignore this error
@@ -55,7 +55,7 @@ def triggerWorkflow(String orgaSlashRepo, String workflowId, Map<String, String>
 	echo "In ${orgaSlashRepo} trigger workflow '${workflowId}' on branch ${referenceBranch} with inputs ${inputs}"
 	def params = ['ref': referenceBranch, 'inputs': inputs]
 	def response = queryGithubAPI('-X POST',"repos/${orgaSlashRepo}/actions/workflows/${workflowId}/dispatches", params, true)
-	if (response?.errors || (response?.status && response.status != 204)) {
+	if (isFailed(response, 204)) {
 		error "Response contains errors:\n${response}"
 	}
 }
@@ -88,4 +88,8 @@ def queryGithubAPI(String method, String endpoint, Map<String, Object> queryPara
 	return readJSON(text: response)
 }
 
+private boolean isFailed(Map response, int successCode) {
+	return response?.errors || (response?.status && response.status?.toInteger() != successCode)
+}
+
 return this

JenkinsJobs/shared/utilities.groovy

@@ -15,7 +15,7 @@ def replaceAllInFile(String filePath, Map<String,String> replacements) {
 	writeFile(file:filePath, text: content)
 }
 
-def commitAllChangesExcludingSubmodules(String commitMessage) {
+def gitCommitAllExcludingSubmodules(String commitMessage) {
 	withEnv(["COMMIT_MESSAGE=${commitMessage}"]) {
 		sh '''
 			#Commit all changes, except for the updated sub-modules here