[RelEng] Unify composite repository creation

JenkinsJobs/Releng/FOLDER.groovy

@@ -58,6 +58,7 @@ pipelineJob('Releng/modifyP2CompositeRepository'){
 The maximum number of childrem the modified composite should contain.
 If the total number of children exceeds this limit (after adding new ones), a corresponding number of children is removed from the beginning of the list.
 ''')
+		stringParam('repositoryName', null, 'Optional name attribute of the composite repository to set (if blank the name is not changed)')
 	}
 	definition {
 		cpsScm {

JenkinsJobs/Releng/modifyP2CompositeRepository.jenkinsfile

@@ -24,9 +24,10 @@ pipeline {
 							repo="https://download.eclipse.org/${repositoryPath}/${repo}"
 						fi
 						echo "Validate content consistency of: ${repo}"
-						${ECLIPSE_EXE} -nosplash -consolelog --launcher.suppressErrors -application org.eclipse.equinox.p2.director -repository "${repo}" -list
+						${ECLIPSE_EXE} -application org.eclipse.equinox.p2.director -repository "${repo}" -list
 					done
 				'''
+				// The resulting composite repository is not tested as it might not exist yet.
 			}
 		}
 		stage('Update composite repository') {
@@ -36,21 +37,25 @@ pipeline {
 			}
 			steps {
 				script {
-					sh "curl -o compositeArtifacts.jar https://download.eclipse.org/${repositoryPath}/compositeArtifacts.jar"
-					def childenXPathSet = sh(script: '''#!/bin/sh +xe
-						unzip -p compositeArtifacts.jar compositeArtifacts.xml |\
-							xmllint - --xpath '/repository/children/child/@location'
-						exit 0
-						''', returnStdout: true).trim()
-					def compositeChildren = 'XPath set is empty' == childenXPathSet ? []
-						: parseList(childenXPathSet, '\\s+')
-							.collect{c -> c.startsWith('location="') && c.endsWith('"') ? c.substring(10, c.length() - 1) : null}
-							.findAll{c -> c != null}
+					def boolean sourceRepositoryExists = sh(script: "curl --fail -o compositeArtifacts.jar https://download.eclipse.org/${repositoryPath}/compositeArtifacts.jar", returnStatus: true) == 0
+					def compositeChildren = []
+					if (sourceRepositoryExists) {
+						def childenXPathSet = sh(script: '''#!/bin/sh +xe
+							unzip -p compositeArtifacts.jar compositeArtifacts.xml |\
+								xmllint - --xpath '/repository/children/child/@location'
+							exit 0
+							''', returnStdout: true).trim()
+						compositeChildren = 'XPath set is empty' == childenXPathSet ? []
+							: parseList(childenXPathSet, '\\s+')
+								.collect{c -> c.startsWith('location="') && c.endsWith('"') ? c.substring(10, c.length() - 1) : null}
+								.findAll{c -> c != null}
+					}
 
 					echo "Current children: ${compositeChildren}"
 					def toAdd = parseList(params.add, ',').unique()
 					def toRemove = parseList(params.remove, ',').unique()
 					toRemove = toRemove.findAll{e -> compositeChildren.contains(e)}
+					def repositoryName = params.repositoryName.trim()
 
 					if (params.sizeLimit) {
 						def sizeRestrictionRemovals = compositeChildren.size() + toAdd.size() - toRemove.size() - Integer.parseInt(params.sizeLimit)
@@ -60,17 +65,19 @@ pipeline {
 					}
 					echo "Children added: ${toAdd}"
 					echo "Children removed: ${toRemove}"
-					if (compositeChildren.size() + toAdd.size() - toRemove.size() == 0) {
+					if (!toRemove.isEmpty() && compositeChildren.size() + toAdd.size() - toRemove.size() == 0) {
 						error('All children are removed and composite repository becomes empty.')
 					}
 					def modifyComposite_xml = """\
 					<?xml version="1.0" encoding="UTF-8"?>
 					<project default="${ANT_TASK_NAME}" basedir=".">
 						<target name="${ANT_TASK_NAME}">
 							<p2.composite.repository>
-								<source location="https://download.eclipse.org/${repositoryPath}" />
-								<repository location="${OUTPUT_PATH}" />
+								<repository location="${OUTPUT_PATH}" ${repositoryName ? ('name="' + repositoryName + '"') :''}/>
 					""".stripIndent()
+					if (sourceRepositoryExists) {
+						modifyComposite_xml +="""			<source location="https://download.eclipse.org/${repositoryPath}" />"""
+					}
 					for (child in toAdd) {
 						modifyComposite_xml += """			<add><repository location="${child}"/></add>\n"""
 					}
@@ -85,12 +92,33 @@ pipeline {
 					writeFile(file: "${ANT_TASK_NAME}.xml", text: modifyComposite_xml)
 
 					sh '''
-						${ECLIPSE_EXE} -nosplash -consolelog --launcher.suppressErrors -debug -data ./eclipse-ws \
+						${ECLIPSE_EXE} -debug -data ./eclipse-ws \
 							-application org.eclipse.ant.core.antRunner -file "${ANT_TASK_NAME}.xml" "${ANT_TASK_NAME}"
 					'''
+					//TODO: Let p2 produce p2.index file
+					writeFile(file: "${OUTPUT_PATH}/p2.index", text: """\
+					#${new Date()}
+					version=1
+					metadata.repository.factory.order=compositeContent.xml,\\!
+					artifact.repository.factory.order=compositeArtifacts.xml,\\!
+					""".stripIndent())
+
+					// The composite files are somehow not compressed if the repository is empty...
+					sh '''
+						cd "${OUTPUT_PATH}"
+						if [[ -f compositeArtifacts.xml ]]; then
+							zip compositeArtifacts.jar compositeArtifacts.xml
+							rm compositeArtifacts.xml
+						fi
+						if [[ -f compositeContent.xml ]]; then
+							zip compositeContent.jar compositeContent.xml
+							rm compositeContent.xml
+						fi
+					'''
 					sshagent(['projects-storage.eclipse.org-bot-ssh']) {
 						sh '''
 							epDownloadsDir='/home/data/httpd/download.eclipse.org'
+							ssh [email protected] mkdir -p "${epDownloadsDir}/${repositoryPath}"
 							scp -r ${OUTPUT_PATH}/* "[email protected]:${epDownloadsDir}/${repositoryPath}"
 						'''
 					}
@@ -115,11 +143,11 @@ def installLatestReleasedEclipsePlatformProduct() {
 	def eclipseURL = sh(script: '''#!/bin/sh +xe
 		source ./buildproperties.txt
 		#TODO: Remove this after the next release!
-		PREVIOUS_RELEASE_ID='S-4.37M1-202507031800'
-		PREVIOUS_RELEASE_VER='4.37M1'
+		PREVIOUS_RELEASE_ID='S-4.37M3-202508141800'
+		PREVIOUS_RELEASE_VER='4.37M3'
 		echo "https://download.eclipse.org/eclipse/downloads/drops4/${PREVIOUS_RELEASE_ID}/eclipse-platform-${PREVIOUS_RELEASE_VER}-linux-gtk-x86_64.tar.gz"
 	''', returnStdout: true).trim()
-	return install('eclipse-platform', eclipseURL) + '/eclipse'
+	return install('eclipse-platform', eclipseURL) + '/eclipse -nosplash --launcher.suppressErrors -consolelog'
 }
 
 def install(String toolType, String url) {

JenkinsJobs/Releng/prepareNextDevCycle.jenkinsfile

@@ -163,28 +163,39 @@ pipeline {
 				commitAllChangesExcludingSubmodules("Move previous version to ${PREVIOUS_RELEASE_CANDIDATE_TAG} in build scripts")
 			}
 		}
-		stage ('Create New Stream Repos') {
+		stage ('Create and update Stream Repos') {
 			when {
 				not { expression { params.DRY_RUN } }
 			}
 			steps {
-				dir("${WORKSPACE}/target/repositories") {
-					sshagent(['projects-storage.eclipse.org-bot-ssh']) {
-						sh '''#!/bin/bash -xe
-							epUpdatesDir='/home/data/httpd/download.eclipse.org/eclipse/updates'
-							templateRepo="${epUpdatesDir}/template_repo"
-
-							ssh [email protected] cp -r ${templateRepo} ${epUpdatesDir}/${NEXT_RELEASE_VERSION}-I-builds
-							ssh [email protected] cp -r ${templateRepo} ${epUpdatesDir}/${NEXT_RELEASE_VERSION}-Y-builds
-							ssh [email protected] cp -r ${templateRepo} ${epUpdatesDir}/${NEXT_RELEASE_VERSION}
-						'''
-					}
-					build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
-						string(name: 'repositoryPath', value: "eclipse/updates/${NEXT_RELEASE_VERSION}-I-builds"),
-						string(name: 'add', value: "https://download.eclipse.org/eclipse/updates/${PREVIOUS_RELEASE_VERSION}-I-builds/${PREVIOUS_RELEASE_CANDIDATE_I_BUILD}/")
-						// Size-limit is not relevant, the repository is initially empty.
-					]
-				}
+				build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
+					string(name: 'repositoryPath', value: "eclipse/updates/${NEXT_RELEASE_VERSION}-I-builds"),
+					string(name: 'repositoryName', value: "Eclipse ${NEXT_RELEASE_VERSION} integration builds"),
+					string(name: 'add', value: "https://download.eclipse.org/eclipse/updates/${PREVIOUS_RELEASE_VERSION}-I-builds/${PREVIOUS_RELEASE_CANDIDATE_I_BUILD}/")
+					// Size-limit is not relevant, the repository is initially empty.
+				]
+				build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
+					string(name: 'repositoryPath', value: "eclipse/updates/${NEXT_RELEASE_VERSION}-Y-builds"),
+					string(name: 'repositoryName', value: "Eclipse ${NEXT_RELEASE_VERSION} Beta Java builds")
+				]
+				build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
+					string(name: 'repositoryPath', value: "eclipse/updates/${NEXT_RELEASE_VERSION}"),
+					string(name: 'repositoryName', value: "Eclipse ${NEXT_RELEASE_VERSION} release")
+				]
+				// Update generic composite repositories for I/Y-builds (clearing all previous children)
+				// Note: The stream number is not in the 'name', because once a 'name' is defined in Eclipse's UI, it does not change.
+				build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
+					string(name: 'repositoryPath', value: "eclipse/updates/I-builds"),
+					string(name: 'repositoryName', value: "Eclipse latest integration builds"),
+					string(name: 'sizeLimit', value: '1'), // Clear all previous children
+					string(name: 'add', value: "https://download.eclipse.org/eclipse/updates/${NEXT_RELEASE_VERSION}-I-builds/"),
+				]
+				build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
+					string(name: 'repositoryPath', value: "eclipse/updates/Y-builds"),
+					string(name: 'repositoryName', value: "Eclipse latest Beta Java builds"),
+					string(name: 'sizeLimit', value: '1'), // Clear all previous children
+					string(name: 'add', value: "https://download.eclipse.org/eclipse/updates/${NEXT_RELEASE_VERSION}-Y-builds/"),
+				]
 			}
 		}
 		stage('Deploy parent-pom and SDK-target') {
@@ -292,13 +303,6 @@ pipeline {
 			}
 		}
 	}
-	post {
-		always {
-			archiveArtifacts allowEmptyArchive: true, artifacts: '\
-				target/repositories/**,\
-				'
-		}
-	}
 }
 
 // --- utility methods

JenkinsJobs/Releng/publishPromotedBuild.jenkinsfile

@@ -66,6 +66,14 @@ pipeline {
 					string(name: 'repositoryPath', value: "eclipse/updates/${RELEASE_VERSION_MAJOR}.${RELEASE_VERSION_MINOR}"),
 					string(name: 'add', value: "${RELEASE_BUILD_ID}")
 				]
+				// Update generic composite repositories for latest release (clearing all previous children)
+				// Note: The stream number is not in the 'name', because once a 'name' is defined in Eclipse's UI, it does not change.
+				build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
+					string(name: 'repositoryPath', value: "eclipse/updates/latest"),
+					string(name: 'repositoryName', value: "Eclipse latest release"),
+					string(name: 'sizeLimit', value: '1'), // Clear all previous children
+					string(name: 'add', value: "https://download.eclipse.org/eclipse/updates/${RELEASE_VERSION_MAJOR}.${RELEASE_VERSION_MINOR}/"),
+				]
 			}
 		}
 	}

RELEASE.md

@@ -206,12 +206,6 @@ Previously they were created in ther own job:
 **General Cleanup**
   - In [eclipse.platform.common] search for and clear out all of the forceQualifierUpdate.txt files.  
     The context here is that the doc builds only check for changes in this repo and so these files need to be changed to trigger a full rebuild.
-* #### **Create Generic Composites**
-   - After First Stable Ibuild move Generic repos to next stream.
-   - Run the [Create Generic Composites](https://ci.eclipse.org/releng/job/Releng/job/createGenericComposites/) job to recreate the generic build repos for the next release. 
-      - `currentStream`: To clarify this is the next stream, not the one currently being released. If you are releasing 4.32, the 'current' stream is 4.33 so that repos are created for it.
-      - `previousStream`: The stream being released, which needs to be removed.  
-      - For reference, the generic repositories created are for the [latest GA release](https://download.eclipse.org/eclipse/updates/latest/) and the current (ongoing) [I-builds](https://download.eclipse.org/eclipse/updates/I-builds/), [Y-builds](https://download.eclipse.org/eclipse/updates/Y-builds/) and [P-builds](https://download.eclipse.org/eclipse/updates/P-builds/). 
 
 **RC2a Release**
   * Sometimes there is a critical issue that requires a fix, if it's decided that one is needed then an RC2a (followed by RC2b, RC2c etc if necessary) is built from the maintenance branch and promoted using the RC2 process.