[Build] Add dedicated job to modify a p2 composite repository

JenkinsJobs/Builds/build.jenkinsfile

@@ -280,6 +280,17 @@ spec:
                         ./mb620_promoteUpdateSite.sh $CJE_ROOT/buildproperties.shsource
                       '''
                   }
+					script {
+						if ("${COMPARATOR_ERRORS_SUBJECT}" != '' && "$BUILD_TYPE" == "I") {
+							echo 'Skip adding unstable build to composite repository.'
+							return
+						}
+						build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
+							string(name: 'repositoryPath', value: "eclipse/updates/${RELEASE_VER}-I-builds"),
+							string(name: 'add', value: "${BUILD_IID}"),
+							string(name: 'sizeLimit', value: '3')
+						]
+					}
 				}
 			}
 		}

JenkinsJobs/Builds/markBuild.groovy

@@ -22,7 +22,10 @@ pipeline {
 	}
 	stages {
 		stage('Mark ''' + marker + ''''){
-			steps { // workspace is not always cleaned by default. Clean before custom tools are installed into workspace.
+			environment {
+				RELEASE_VER = readBuildProperty('RELEASE_VER')
+			}
+			steps {
 				sshagent(['projects-storage.eclipse.org-bot-ssh']) {
 					sh \'''#!/bin/bash -xe
 						# this function executes command passed as command line parameter and 
@@ -40,48 +43,8 @@ pipeline {
 							echo "IssueUrl is empty! Exiting."
 							exit 1
 						fi''' : '') + '''
-
 						epDownloadDir=/home/data/httpd/download.eclipse.org/eclipse
-						dropsPath=${epDownloadDir}/downloads/drops4
-						p2RepoPath=${epDownloadDir}/updates
-						buildDir=${dropsPath}/${buildId}
-
-						workingDir=${epDownloadDir}/workingDir
-
-						workspace=${workingDir}/${JOB_NAME}-${BUILD_NUMBER}
-
-						ssh [email protected] rm -rf ${workingDir}/${JOB_NAME}*
-
-						ssh [email protected] mkdir -p ${workspace}
-						ssh [email protected] cd ${workspace}
-
-						#get latest Eclipse platform product
-						epRelDir=$(ssh [email protected] ls -d --format=single-column ${dropsPath}/R-*|sort|tail -1)
-						ssh [email protected] tar -C ${workspace} -xzf ${epRelDir}/eclipse-platform-*-linux-gtk-x86_64.tar.gz
-
-						#get requisite tools
-						markingScriptName=\'''' + { switch (marker) {
-							case 'stable': return 'addToComposite'
-							case 'unstable': return 'removeFromComposite'
-							default : throw new IllegalArgumentException("Unknown marker: ${marker}")
-						} }() + ''''
-						ssh [email protected] wget -O ${workspace}/${markingScriptName}.xml https://download.eclipse.org/eclipse/relengScripts/cje-production/scripts/${markingScriptName}.xml
-
-						#triggering ant runner
-						baseBuilderDir=${workspace}/eclipse
-						javaCMD=/opt/public/common/java/openjdk/jdk-21_x64-latest/bin/java
-
-						launcherJar=$(ssh [email protected] find ${baseBuilderDir}/. -name "org.eclipse.equinox.launcher_*.jar" | sort | head -1 )
-
-						scp [email protected]:${buildDir}/buildproperties.shsource .
-						source ./buildproperties.shsource
-						repoDir=/home/data/httpd/download.eclipse.org/eclipse/updates/${STREAMMajor}.${STREAMMinor}-${BUILD_TYPE}-builds
-
-						devworkspace=${workspace}/workspace-antRunner
-						devArgs=-Xmx512m
-						extraArgs="${markingScriptName} -Drepodir=${repoDir} -Dcomplocation=${buildId}"
-						ssh [email protected]  ${javaCMD} -jar ${launcherJar} -nosplash -consolelog -debug -data $devworkspace \\
-							-application org.eclipse.ant.core.antRunner -file ${workspace}/${markingScriptName}.xml ${extraArgs} -vmargs $devArgs
+						buildDir="${epDownloadDir}/downloads/drops4/${buildId}"
 						''' + { switch (marker) {
 							case 'stable': return '''
 						#Remove hidden attribute and unstable tags
@@ -98,14 +61,26 @@ pipeline {
 						'''
 							default : throw new IllegalArgumentException("Unknown marker: ${marker}")
 						} }() + '''
-						ssh [email protected] rm -rf ${workingDir}/${JOB_NAME}*
 					\'''
-					build job: 'Releng/updateIndex', wait: false
 				}
+				build job: 'Releng/updateIndex', wait: false
+				build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
+					string(name: 'repositoryPath', value: "eclipse/updates/${RELEASE_VER}-I-builds"),
+					string(name: \'''' + { switch (marker) {
+						case 'stable': return 'add'
+						case 'unstable': return 'remove'
+						default : throw new IllegalArgumentException("Unknown marker: ${marker}")
+					} }() + '''\', value: "${buildId}")
+				]
 			}
 		}
 	}
 }
+
+def readBuildProperty(String name) {
+	def value = sh(script: """curl https://download.eclipse.org/eclipse/downloads/drops4/${buildId}/buildproperties.properties | grep "^${name}" | cut -d'=' -f2""", returnStdout: true).trim()
+	return value.startsWith('"') && value.endsWith('"') ? value.substring(1, value.length() - 1) : value
+}
 	''')
 			}
 		}

JenkinsJobs/Releng/FOLDER.groovy

@@ -61,6 +61,29 @@ Releases are published to <a href="https://repo1.maven.org/maven2/org/eclipse/">
 	}
 }
 
+pipelineJob('Releng/modifyP2CompositeRepository'){
+	displayName('Modify P2 composite repository')
+	description('Add or remove children from an Eclipse-P2 composite repository.')
+	parameters {
+		stringParam('repositoryPath', null, "Relative repository path from https://download.eclipse.org/. E.g. 'eclipse/updates/4.37-I-builds'")
+		stringParam('add', null, 'Comma separated list of children to add. May be empty')
+		stringParam('remove', null, 'Comma separated list of children to remove. May be empty')
+		stringParam('sizeLimit', null, '''
+The maximum number of childrem the modified composite should contain.
+If the total number of children exceeds this limit (after adding new ones), a corresponding number of children is removed from the beginning of the list.
+''')
+	}
+	definition {
+		cpsScm {
+			lightweight(true)
+			scm {
+				github('eclipse-platform/eclipse.platform.releng.aggregator', 'master')
+			}
+			scriptPath('JenkinsJobs/Releng/modifyP2CompositeRepository.jenkinsfile')
+		}
+	}
+}
+
 pipelineJob('Releng/prepareNextDevCycle'){
 	displayName('Prepare Next Development Cycle')
 	description('Perform all steps to prepare the next development cycle of Eclipse.')

JenkinsJobs/Releng/makeVisible.jenkinsfile

@@ -62,42 +62,10 @@ pipeline {
 				environment name: 'RELEASE_TYPE', value: 'R'
 			}
 			steps {
-				sshagent(['projects-storage.eclipse.org-bot-ssh']) {
-					sh '''
-						#Repository will be available only for R builds. add it to composite
-						epDownloadDir=/home/data/httpd/download.eclipse.org/eclipse
-						dropsPath=${epDownloadDir}/downloads/drops4
-						workingDir=${epDownloadDir}/workingDir
-
-						workspace=${workingDir}/${JOB_NAME}-${BUILD_NUMBER}
-
-						ssh [email protected] rm -rf ${workingDir}/${JOB_NAME}*
-
-						ssh [email protected] mkdir -p ${workspace}
-						ssh [email protected] cd ${workspace}
-
-						#get latest Eclipse platform product
-						epRelDir=$(ssh [email protected] ls -d --format=single-column ${dropsPath}/R-*|sort|tail -1)
-						ssh [email protected] tar -C ${workspace} -xzf ${epRelDir}/eclipse-platform-*-linux-gtk-x86_64.tar.gz
-
-						#get requisite tools
-						ssh [email protected] wget -O ${workspace}/addToComposite.xml https://download.eclipse.org/eclipse/relengScripts/cje-production/scripts/addToComposite.xml
-
-						#triggering ant runner
-						baseBuilderDir=${workspace}/eclipse
-						javaCMD=/opt/public/common/java/openjdk/jdk-21_x64-latest/bin/java
-						launcherJar=$(ssh [email protected] find ${baseBuilderDir}/. -name "org.eclipse.equinox.launcher_*.jar" | sort | head -1 )
-
-						repoDir=/"home/data/httpd/download.eclipse.org/eclipse/updates/${BUILD_MAJOR}.${BUILD_MINOR}"
-
-						devWorkspace=${workspace}/workspace-antRunner
-						devArgs=-Xmx512m
-						extraArgs="addToComposite -Drepodir=${repoDir} -Dcomplocation=${RELEASE_ID}"
-						ssh [email protected]  ${javaCMD} -jar ${launcherJar} -nosplash -consolelog -debug -data $devWorkspace -application org.eclipse.ant.core.antRunner -file ${workspace}/addToComposite.xml ${extraArgs} -vmargs $devArgs
-
-						ssh [email protected] rm -rf ${workingDir}/${JOB_NAME}*
-					'''
-				}
+				build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [
+					string(name: 'repositoryPath', value: "eclipse/updates/${RELEASE_VERSION_MAJOR}.${RELEASE_VERSION_MINOR}"),
+					string(name: 'add', value: "${RELEASE_ID}")
+				]
 			}
 		}
 	}

JenkinsJobs/Releng/modifyP2CompositeRepository.jenkinsfile

@@ -0,0 +1,130 @@
+pipeline {
+	options {
+		disableConcurrentBuilds() // prevent concurrent updates of the same repository
+		timestamps()
+		skipDefaultCheckout()
+		timeout(time: 15, unit: 'MINUTES')
+		buildDiscarder(logRotator(numToKeepStr:'5'))
+	}
+	agent {
+		label 'ubuntu-2404'
+	}
+	tools {
+		jdk 'temurin-jdk21-latest'
+	}
+	environment {
+		ECLIPSE_EXE = installLatestReleasedEclipsePlatformProduct()
+	}
+	stages {
+		stage('Check Composites Validity') {
+			steps {
+				sh '''
+					for repo in ${add//,/ }; do
+						if [[ $repo != http* ]]; then
+							repo="https://download.eclipse.org/${repositoryPath}/${repo}"
+						fi
+						echo "Validate content consistency of: ${repo}"
+						${ECLIPSE_EXE} -nosplash -consolelog --launcher.suppressErrors -application org.eclipse.equinox.p2.director -repository "${repo}" -list
+					done
+				'''
+			}
+		}
+		stage('Update composite repository') {
+			environment {
+				ANT_TASK_NAME = 'modifyComposite'
+				OUTPUT_PATH = 'output-repository'
+			}
+			steps {
+				script {
+					sh "curl -o compositeArtifacts.jar https://download.eclipse.org/${repositoryPath}/compositeArtifacts.jar"
+					def childenXPathSet = sh(script: '''#!/bin/sh +xe
+						unzip -p compositeArtifacts.jar compositeArtifacts.xml |\
+							xmllint - --xpath '/repository/children/child/@location'
+						exit 0
+						''', returnStdout: true).trim()
+					def compositeChildren = 'XPath set is empty' == childenXPathSet ? []
+						: parseList(childenXPathSet, '\\s+')
+							.collect{c -> c.startsWith('location="') && c.endsWith('"') ? c.substring(10, c.length() - 1) : null}
+							.findAll{c -> c != null}
+
+					echo "Current children: ${compositeChildren}"
+					def toAdd = parseList(params.add, ',').unique()
+					def toRemove = parseList(params.remove, ',').unique()
+					toRemove = toRemove.findAll{e -> compositeChildren.contains(e)}
+
+					if (params.sizeLimit) {
+						def sizeRestrictionRemovals = compositeChildren.size() + toAdd.size() - toRemove.size() - Integer.parseInt(params.sizeLimit)
+						if (sizeRestrictionRemovals > 0) {
+							toRemove += compositeChildren.subList(0, sizeRestrictionRemovals)
+						}
+					}
+					echo "Children added: ${toAdd}"
+					echo "Children removed: ${toRemove}"
+					if (compositeChildren.size() + toAdd.size() - toRemove.size() == 0) {
+						error('All children are removed and composite repository becomes empty.')
+					}
+					def modifyComposite_xml = """\
+					<?xml version="1.0" encoding="UTF-8"?>
+					<project default="${ANT_TASK_NAME}" basedir=".">
+						<target name="${ANT_TASK_NAME}">
+							<p2.composite.repository>
+								<source location="https://download.eclipse.org/${repositoryPath}" />
+								<repository location="${OUTPUT_PATH}" />
+					""".stripIndent()
+					for (child in toAdd) {
+						modifyComposite_xml += """			<add><repository location="${child}"/></add>\n"""
+					}
+					for (child in toRemove) {
+						modifyComposite_xml += """			<remove><repository location="${child}"/></remove>\n"""
+					}
+					modifyComposite_xml+= '''\
+							</p2.composite.repository>
+						</target>
+					</project>
+					'''.stripIndent()
+					writeFile(file: "${ANT_TASK_NAME}.xml", text: modifyComposite_xml)
+
+					sh '''
+						${ECLIPSE_EXE} -nosplash -consolelog --launcher.suppressErrors -debug -data ./eclipse-ws \
+							-application org.eclipse.ant.core.antRunner -file "${ANT_TASK_NAME}.xml" "${ANT_TASK_NAME}"
+					'''
+					sshagent(['projects-storage.eclipse.org-bot-ssh']) {
+						sh '''
+							epDownloadsDir='/home/data/httpd/download.eclipse.org'
+							scp -r ${OUTPUT_PATH}/* "[email protected]:${epDownloadsDir}/${repositoryPath}"
+						'''
+					}
+				}
+			}
+		}
+	}
+	post {
+		always {
+			archiveArtifacts allowEmptyArchive: true, artifacts: '**/*', excludes: 'tools/**/*'
+		}
+	}
+}
+
+@NonCPS
+def parseList(String str, String delimiterPattern) {
+	return str !=null && !str.trim().isEmpty() ? str.trim().split(delimiterPattern).collect{c -> c.trim()} : []
+}
+
+def installLatestReleasedEclipsePlatformProduct() {
+	sh 'curl -o buildproperties.txt https://download.eclipse.org/eclipse/relengScripts/cje-production/buildproperties.txt'
+	def eclipseURL = sh(script: '''#!/bin/sh +xe
+		source ./buildproperties.txt
+		#TODO: Remove this after the next release!
+		PREVIOUS_RELEASE_ID='S-4.37M1-202507031800'
+		PREVIOUS_RELEASE_VER='4.37M1'
+		echo "https://download.eclipse.org/eclipse/downloads/drops4/${PREVIOUS_RELEASE_ID}/eclipse-platform-${PREVIOUS_RELEASE_VER}-linux-gtk-x86_64.tar.gz"
+	''', returnStdout: true).trim()
+	return install('eclipse-platform', eclipseURL) + '/eclipse'
+}
+
+def install(String toolType, String url) {
+	dir("${WORKSPACE}/tools/${toolType}") {
+		sh "curl -L ${url} | tar -xzf -"
+		return "${pwd()}/" + sh(script: 'ls', returnStdout: true).trim()
+	}
+}