diff --git a/JenkinsJobs/Releng/FOLDER.groovy b/JenkinsJobs/Releng/FOLDER.groovy index 23968939991..97a4c0f5256 100644 --- a/JenkinsJobs/Releng/FOLDER.groovy +++ b/JenkinsJobs/Releng/FOLDER.groovy @@ -58,6 +58,7 @@ pipelineJob('Releng/modifyP2CompositeRepository'){ The maximum number of childrem the modified composite should contain. If the total number of children exceeds this limit (after adding new ones), a corresponding number of children is removed from the beginning of the list. ''') + stringParam('repositoryName', null, 'Optional name attribute of the composite repository to set (if blank the name is not changed)') } definition { cpsScm { diff --git a/JenkinsJobs/Releng/createGenericComposites.groovy b/JenkinsJobs/Releng/createGenericComposites.groovy deleted file mode 100644 index 246328c0189..00000000000 --- a/JenkinsJobs/Releng/createGenericComposites.groovy +++ /dev/null @@ -1,58 +0,0 @@ -job('Releng/createGenericComposites'){ - displayName('Create Generic Composites') - description('Remove generic repositories (I-builds etc) for the previous stream and create new versions for the new stream.') - - logRotator { - daysToKeep(5) - numToKeep(5) - } - - parameters { - stringParam('currentStream', null, 'Current release stream, for example: 4.25') - stringParam('previousStream', null, 'Previous release stream, for example: 4.24') - } - - jdk('openjdk-jdk11-latest') - - label('basic') - - wrappers { //adds pre/post actions - timestamps() - sshAgent('projects-storage.eclipse.org-bot-ssh') - } - - steps { - shell(''' -#!/bin/bash -x - -epDownloadDir=/home/data/httpd/download.eclipse.org/eclipse -dropsPath=${epDownloadDir}/downloads/drops4 - -cd ${WORKSPACE} -epRelDir=$(ssh genie.releng@projects-storage.eclipse.org ls -d --format=single-column ${dropsPath}/R-*|sort|tail -1) -scp genie.releng@projects-storage.eclipse.org:${epRelDir}/eclipse-SDK-*-linux-gtk-x86_64.tar.gz eclipse-SDK.tar.gz - -tar xvzf eclipse-SDK.tar.gz - -ECLIPSE_EXE=${WORKSPACE}/eclipse/eclipse - - -# get the update script -wget https://download.eclipse.org/eclipse/relengScripts/cje-production/scripts/updateGenericComposites.xml - -${ECLIPSE_EXE} --launcher.suppressErrors -nosplash -console -data workspace-updateGenericComposite -application org.eclipse.ant.core.antRunner -f updateGenericComposites.xml ${extraArgs} -vmargs -DcurrentStream=${currentStream} -DpreviousStream=${previousStream} -Dworkspace=${WORKSPACE} - -# remove existing generic repos -ssh genie.releng@projects-storage.eclipse.org rm -rf ${epDownloadDir}/updates/I-builds -ssh genie.releng@projects-storage.eclipse.org rm -rf ${epDownloadDir}/updates/P-builds -ssh genie.releng@projects-storage.eclipse.org rm -rf ${epDownloadDir}/updates/Y-builds -ssh genie.releng@projects-storage.eclipse.org rm -rf ${epDownloadDir}/updates/latest - -#copy newly created generic repos -scp -r I-builds genie.releng@projects-storage.eclipse.org:${epDownloadDir}/updates/. -scp -r P-builds genie.releng@projects-storage.eclipse.org:${epDownloadDir}/updates/. -scp -r Y-builds genie.releng@projects-storage.eclipse.org:${epDownloadDir}/updates/. -scp -r latest genie.releng@projects-storage.eclipse.org:${epDownloadDir}/updates/. - ''') - } -} diff --git a/JenkinsJobs/Releng/modifyP2CompositeRepository.jenkinsfile b/JenkinsJobs/Releng/modifyP2CompositeRepository.jenkinsfile index 35d861f4958..2de2a0fa6d0 100644 --- a/JenkinsJobs/Releng/modifyP2CompositeRepository.jenkinsfile +++ b/JenkinsJobs/Releng/modifyP2CompositeRepository.jenkinsfile @@ -24,9 +24,10 @@ pipeline { repo="https://download.eclipse.org/${repositoryPath}/${repo}" fi echo "Validate content consistency of: ${repo}" - ${ECLIPSE_EXE} -nosplash -consolelog --launcher.suppressErrors -application org.eclipse.equinox.p2.director -repository "${repo}" -list + ${ECLIPSE_EXE} -application org.eclipse.equinox.p2.director -repository "${repo}" -list done ''' + // The resulting composite repository is not tested as it might not exist yet. } } stage('Update composite repository') { @@ -36,21 +37,25 @@ pipeline { } steps { script { - sh "curl -o compositeArtifacts.jar https://download.eclipse.org/${repositoryPath}/compositeArtifacts.jar" - def childenXPathSet = sh(script: '''#!/bin/sh +xe - unzip -p compositeArtifacts.jar compositeArtifacts.xml |\ - xmllint - --xpath '/repository/children/child/@location' - exit 0 - ''', returnStdout: true).trim() - def compositeChildren = 'XPath set is empty' == childenXPathSet ? [] - : parseList(childenXPathSet, '\\s+') - .collect{c -> c.startsWith('location="') && c.endsWith('"') ? c.substring(10, c.length() - 1) : null} - .findAll{c -> c != null} + def boolean sourceRepositoryExists = sh(script: "curl --fail -o compositeArtifacts.jar https://download.eclipse.org/${repositoryPath}/compositeArtifacts.jar", returnStatus: true) == 0 + def compositeChildren = [] + if (sourceRepositoryExists) { + def childenXPathSet = sh(script: '''#!/bin/sh +xe + unzip -p compositeArtifacts.jar compositeArtifacts.xml |\ + xmllint - --xpath '/repository/children/child/@location' + exit 0 + ''', returnStdout: true).trim() + compositeChildren = 'XPath set is empty' == childenXPathSet ? [] + : parseList(childenXPathSet, '\\s+') + .collect{c -> c.startsWith('location="') && c.endsWith('"') ? c.substring(10, c.length() - 1) : null} + .findAll{c -> c != null} + } echo "Current children: ${compositeChildren}" def toAdd = parseList(params.add, ',').unique() def toRemove = parseList(params.remove, ',').unique() toRemove = toRemove.findAll{e -> compositeChildren.contains(e)} + def repositoryName = params.repositoryName.trim() if (params.sizeLimit) { def sizeRestrictionRemovals = compositeChildren.size() + toAdd.size() - toRemove.size() - Integer.parseInt(params.sizeLimit) @@ -60,7 +65,7 @@ pipeline { } echo "Children added: ${toAdd}" echo "Children removed: ${toRemove}" - if (compositeChildren.size() + toAdd.size() - toRemove.size() == 0) { + if (!toRemove.isEmpty() && compositeChildren.size() + toAdd.size() - toRemove.size() == 0) { error('All children are removed and composite repository becomes empty.') } def modifyComposite_xml = """\ @@ -68,9 +73,11 @@ pipeline { - - + """.stripIndent() + if (sourceRepositoryExists) { + modifyComposite_xml +=""" """ + } for (child in toAdd) { modifyComposite_xml += """ \n""" } @@ -85,12 +92,33 @@ pipeline { writeFile(file: "${ANT_TASK_NAME}.xml", text: modifyComposite_xml) sh ''' - ${ECLIPSE_EXE} -nosplash -consolelog --launcher.suppressErrors -debug -data ./eclipse-ws \ + ${ECLIPSE_EXE} -debug -data ./eclipse-ws \ -application org.eclipse.ant.core.antRunner -file "${ANT_TASK_NAME}.xml" "${ANT_TASK_NAME}" ''' + //TODO: Let p2 produce p2.index file + writeFile(file: "${OUTPUT_PATH}/p2.index", text: """\ + #${new Date()} + version=1 + metadata.repository.factory.order=compositeContent.xml,\\! + artifact.repository.factory.order=compositeArtifacts.xml,\\! + """.stripIndent()) + + // The composite files are somehow not compressed if the repository is empty... + sh ''' + cd "${OUTPUT_PATH}" + if [[ -f compositeArtifacts.xml ]]; then + zip compositeArtifacts.jar compositeArtifacts.xml + rm compositeArtifacts.xml + fi + if [[ -f compositeContent.xml ]]; then + zip compositeContent.jar compositeContent.xml + rm compositeContent.xml + fi + ''' sshagent(['projects-storage.eclipse.org-bot-ssh']) { sh ''' epDownloadsDir='/home/data/httpd/download.eclipse.org' + ssh genie.releng@projects-storage.eclipse.org mkdir -p "${epDownloadsDir}/${repositoryPath}" scp -r ${OUTPUT_PATH}/* "genie.releng@projects-storage.eclipse.org:${epDownloadsDir}/${repositoryPath}" ''' } @@ -115,11 +143,11 @@ def installLatestReleasedEclipsePlatformProduct() { def eclipseURL = sh(script: '''#!/bin/sh +xe source ./buildproperties.txt #TODO: Remove this after the next release! - PREVIOUS_RELEASE_ID='S-4.37M1-202507031800' - PREVIOUS_RELEASE_VER='4.37M1' + PREVIOUS_RELEASE_ID='S-4.37M3-202508141800' + PREVIOUS_RELEASE_VER='4.37M3' echo "https://download.eclipse.org/eclipse/downloads/drops4/${PREVIOUS_RELEASE_ID}/eclipse-platform-${PREVIOUS_RELEASE_VER}-linux-gtk-x86_64.tar.gz" ''', returnStdout: true).trim() - return install('eclipse-platform', eclipseURL) + '/eclipse' + return install('eclipse-platform', eclipseURL) + '/eclipse -nosplash --launcher.suppressErrors -consolelog' } def install(String toolType, String url) { diff --git a/JenkinsJobs/Releng/prepareNextDevCycle.jenkinsfile b/JenkinsJobs/Releng/prepareNextDevCycle.jenkinsfile index fe0253c987e..3066914e3d7 100644 --- a/JenkinsJobs/Releng/prepareNextDevCycle.jenkinsfile +++ b/JenkinsJobs/Releng/prepareNextDevCycle.jenkinsfile @@ -163,28 +163,39 @@ pipeline { commitAllChangesExcludingSubmodules("Move previous version to ${PREVIOUS_RELEASE_CANDIDATE_TAG} in build scripts") } } - stage ('Create New Stream Repos') { + stage ('Create and update Stream Repos') { when { not { expression { params.DRY_RUN } } } steps { - dir("${WORKSPACE}/target/repositories") { - sshagent(['projects-storage.eclipse.org-bot-ssh']) { - sh '''#!/bin/bash -xe - epUpdatesDir='/home/data/httpd/download.eclipse.org/eclipse/updates' - templateRepo="${epUpdatesDir}/template_repo" - - ssh genie.releng@projects-storage.eclipse.org cp -r ${templateRepo} ${epUpdatesDir}/${NEXT_RELEASE_VERSION}-I-builds - ssh genie.releng@projects-storage.eclipse.org cp -r ${templateRepo} ${epUpdatesDir}/${NEXT_RELEASE_VERSION}-Y-builds - ssh genie.releng@projects-storage.eclipse.org cp -r ${templateRepo} ${epUpdatesDir}/${NEXT_RELEASE_VERSION} - ''' - } - build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [ - string(name: 'repositoryPath', value: "eclipse/updates/${NEXT_RELEASE_VERSION}-I-builds"), - string(name: 'add', value: "https://download.eclipse.org/eclipse/updates/${PREVIOUS_RELEASE_VERSION}-I-builds/${PREVIOUS_RELEASE_CANDIDATE_I_BUILD}/") - // Size-limit is not relevant, the repository is initially empty. - ] - } + build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [ + string(name: 'repositoryPath', value: "eclipse/updates/${NEXT_RELEASE_VERSION}-I-builds"), + string(name: 'repositoryName', value: "Eclipse ${NEXT_RELEASE_VERSION} integration builds"), + string(name: 'add', value: "https://download.eclipse.org/eclipse/updates/${PREVIOUS_RELEASE_VERSION}-I-builds/${PREVIOUS_RELEASE_CANDIDATE_I_BUILD}/") + // Size-limit is not relevant, the repository is initially empty. + ] + build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [ + string(name: 'repositoryPath', value: "eclipse/updates/${NEXT_RELEASE_VERSION}-Y-builds"), + string(name: 'repositoryName', value: "Eclipse ${NEXT_RELEASE_VERSION} Beta Java builds") + ] + build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [ + string(name: 'repositoryPath', value: "eclipse/updates/${NEXT_RELEASE_VERSION}"), + string(name: 'repositoryName', value: "Eclipse ${NEXT_RELEASE_VERSION} release") + ] + // Update generic composite repositories for I/Y-builds (clearing all previous children) + // Note: The stream number is not in the 'name', because once a 'name' is defined in Eclipse's UI, it does not change. + build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [ + string(name: 'repositoryPath', value: "eclipse/updates/I-builds"), + string(name: 'repositoryName', value: "Eclipse latest integration builds"), + string(name: 'sizeLimit', value: '1'), // Clear all previous children + string(name: 'add', value: "https://download.eclipse.org/eclipse/updates/${NEXT_RELEASE_VERSION}-I-builds/"), + ] + build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [ + string(name: 'repositoryPath', value: "eclipse/updates/Y-builds"), + string(name: 'repositoryName', value: "Eclipse latest Beta Java builds"), + string(name: 'sizeLimit', value: '1'), // Clear all previous children + string(name: 'add', value: "https://download.eclipse.org/eclipse/updates/${NEXT_RELEASE_VERSION}-Y-builds/"), + ] } } stage('Deploy parent-pom and SDK-target') { @@ -292,13 +303,6 @@ pipeline { } } } - post { - always { - archiveArtifacts allowEmptyArchive: true, artifacts: '\ - target/repositories/**,\ - ' - } - } } // --- utility methods diff --git a/JenkinsJobs/Releng/publishPromotedBuild.jenkinsfile b/JenkinsJobs/Releng/publishPromotedBuild.jenkinsfile index 84f6879eed7..05e325b20cf 100644 --- a/JenkinsJobs/Releng/publishPromotedBuild.jenkinsfile +++ b/JenkinsJobs/Releng/publishPromotedBuild.jenkinsfile @@ -66,6 +66,14 @@ pipeline { string(name: 'repositoryPath', value: "eclipse/updates/${RELEASE_VERSION_MAJOR}.${RELEASE_VERSION_MINOR}"), string(name: 'add', value: "${RELEASE_BUILD_ID}") ] + // Update generic composite repositories for latest release (clearing all previous children) + // Note: The stream number is not in the 'name', because once a 'name' is defined in Eclipse's UI, it does not change. + build job: 'Releng/modifyP2CompositeRepository', wait: true, propagate: true, parameters: [ + string(name: 'repositoryPath', value: "eclipse/updates/latest"), + string(name: 'repositoryName', value: "Eclipse latest release"), + string(name: 'sizeLimit', value: '1'), // Clear all previous children + string(name: 'add', value: "https://download.eclipse.org/eclipse/updates/${RELEASE_VERSION_MAJOR}.${RELEASE_VERSION_MINOR}/"), + ] } } } diff --git a/RELEASE.md b/RELEASE.md index f47a89b7ac5..bcfa230a608 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -206,12 +206,6 @@ Previously they were created in ther own job: **General Cleanup** - In [eclipse.platform.common] search for and clear out all of the forceQualifierUpdate.txt files. The context here is that the doc builds only check for changes in this repo and so these files need to be changed to trigger a full rebuild. -* #### **Create Generic Composites** - - After First Stable Ibuild move Generic repos to next stream. - - Run the [Create Generic Composites](https://ci.eclipse.org/releng/job/Releng/job/createGenericComposites/) job to recreate the generic build repos for the next release. - - `currentStream`: To clarify this is the next stream, not the one currently being released. If you are releasing 4.32, the 'current' stream is 4.33 so that repos are created for it. - - `previousStream`: The stream being released, which needs to be removed. - - For reference, the generic repositories created are for the [latest GA release](https://download.eclipse.org/eclipse/updates/latest/) and the current (ongoing) [I-builds](https://download.eclipse.org/eclipse/updates/I-builds/), [Y-builds](https://download.eclipse.org/eclipse/updates/Y-builds/) and [P-builds](https://download.eclipse.org/eclipse/updates/P-builds/). **RC2a Release** * Sometimes there is a critical issue that requires a fix, if it's decided that one is needed then an RC2a (followed by RC2b, RC2c etc if necessary) is built from the maintenance branch and promoted using the RC2 process. diff --git a/cje-production/scripts/updateGenericComposites.xml b/cje-production/scripts/updateGenericComposites.xml deleted file mode 100644 index 2e681e84b8f..00000000000 --- a/cje-production/scripts/updateGenericComposites.xml +++ /dev/null @@ -1,107 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -