From eb2da4803979825c1cf32c8b318bc86d6548eefe Mon Sep 17 00:00:00 2001 From: Hannes Wellmann Date: Mon, 23 Jun 2025 19:16:55 +0200 Subject: [PATCH] [Build] Remove old builds from composite p2-repo when adding new ones Previously the I/Y-builds only added the just built p2-repository to their corresponding composite repository and a separate job, running twice a day, has trimmed down that repositorie's content afterwards to retain only two children. This made the setup's and job configurations more cumbersome and the repository content less predictable. Furthermore the the pruning only removed children that where located directly within the composite's directory and not absolute references to other repositories. This simplifies this task by removing the oldest children while adding the new, just built one, making sure the composite contains at most three children. Permanently fixes - https://github.com/eclipse-platform/eclipse.platform.releng.aggregator/issues/3123 --- JenkinsJobs/Cleanup/pruneDailyRepos.groovy | 70 ----- cje-production/cleaners/cleanupNightlyRepo.sh | 242 ------------------ .../mbscripts/mb620_promoteUpdateSite.sh | 31 ++- cje-production/scripts/addToComposite.xml | 3 +- 4 files changed, 31 insertions(+), 315 deletions(-) delete mode 100644 JenkinsJobs/Cleanup/pruneDailyRepos.groovy delete mode 100755 cje-production/cleaners/cleanupNightlyRepo.sh diff --git a/JenkinsJobs/Cleanup/pruneDailyRepos.groovy b/JenkinsJobs/Cleanup/pruneDailyRepos.groovy deleted file mode 100644 index 6df2bb72685..00000000000 --- a/JenkinsJobs/Cleanup/pruneDailyRepos.groovy +++ /dev/null @@ -1,70 +0,0 @@ -job('Cleanup/pruneDailyRepos'){ - displayName('Daily Repo Pruner') - description(''' -This job runs several types of "cleanup" on the build machine and downloads server to remove old builds and other left overs from old build. -It acts as a simple cron job, currently running at 16:00 every day, to execute -.../sdk/cleaners/dailyCleanBuildMachine.sh -and other such scripts. - ''') - - logRotator { - daysToKeep(10) - numToKeep(5) - } - - jdk('temurin-jdk21-latest') - - label('basic') - - triggers { - cron(''' -0 5 * * * -0 17 * * * - ''') - } - - wrappers { //adds pre/post actions - timestamps() - preBuildCleanup() - sshAgent('projects-storage.eclipse.org-bot-ssh') - timeout { - absolute(30) - } - } - - steps { - shell(''' -#!/bin/bash -x - -epDownloadDir=/home/data/httpd/download.eclipse.org/eclipse -dropsPath=${epDownloadDir}/downloads/drops4 -p2RepoPath=${epDownloadDir}/updates -buildDir=${dropsPath}/${buildId} - -workingDir=${epDownloadDir}/workingDir - -workspace=${workingDir}/${JOB_NAME}-${BUILD_NUMBER} - -ssh genie.releng@projects-storage.eclipse.org rm -rf ${workingDir}/${JOB_NAME}* - -ssh genie.releng@projects-storage.eclipse.org mkdir -p ${workspace} -ssh genie.releng@projects-storage.eclipse.org cd ${workspace} - -#get latest Eclipse platform product -epRelDir=$(ssh genie.releng@projects-storage.eclipse.org ls -d --format=single-column ${dropsPath}/R-*|sort|tail -1) -ssh genie.releng@projects-storage.eclipse.org tar -C ${workspace} -xzf ${epRelDir}/eclipse-platform-*-linux-gtk-x86_64.tar.gz - -ssh genie.releng@projects-storage.eclipse.org wget -O ${workspace}/cleanupNightlyRepo.sh https://raw.githubusercontent.com/eclipse-platform/eclipse.platform.releng.aggregator/master/cje-production/cleaners/cleanupNightlyRepo.sh - -ssh genie.releng@projects-storage.eclipse.org bash -x ${workspace}/cleanupNightlyRepo.sh ${workspace} - -ssh genie.releng@projects-storage.eclipse.org rm -rf ${workingDir}/${JOB_NAME}* - ''') - } - - publishers { - extendedEmail { - recipientList("rahul.mohanan@ibm.com") - } - } -} diff --git a/cje-production/cleaners/cleanupNightlyRepo.sh b/cje-production/cleaners/cleanupNightlyRepo.sh deleted file mode 100755 index b52d38958f7..00000000000 --- a/cje-production/cleaners/cleanupNightlyRepo.sh +++ /dev/null @@ -1,242 +0,0 @@ -#!/bin/bash - -#******************************************************************************* -# Copyright (c) 2025 IBM Corporation and others. -# -# This program and the accompanying materials -# are made available under the terms of the Eclipse Public License 2.0 -# which accompanies this distribution, and is available at -# https://www.eclipse.org/legal/epl-2.0/ -# -# SPDX-License-Identifier: EPL-2.0 -# -# Contributors: -# IBM Corporation - initial API and implementation -#******************************************************************************* - -function writeHeader () -{ - compositeRepoDir="$1" - antBuildFile=$2 - if [[ -z "${compositeRepoDir}" ]] - then - echo -e "\n\tWARNING: compositeRepoDir not passed to writeHeader function as expected. But will continue with variable for later use?" - compositeRepoDir="\$\{compositeRepoDir\}" - fi - echo -e "" > $antBuildFile - echo -e "> $antBuildFile - echo -e " basedir=\".\"" >>$antBuildFile - echo -e " default=\"cleanup\">" >>$antBuildFile - echo -e " " >>$antBuildFile - echo -e " " >>$antBuildFile - echo -e " " >>$antBuildFile - echo -e " " >> $antBuildFile -} - -function writeReposToRemove () -{ - antBuildFile=$1 - for repo in "${reposToRemove[@]}" - do - echo " " >> $antBuildFile - done - -} - -function writeClosing () -{ - antBuildFile=$1 - echo -e " " >> $antBuildFile - echo -e " " >> $antBuildFile - echo -e " " >> $antBuildFile - echo -e "" >> $antBuildFile -} - -function generateCleanupXML () -{ - mainRepoDir=$1 - antBuildFile=$2 - if [[ -z "${mainRepoDir}" || ! -e "${mainRepoDir}" ]] - then - echo -e "\n\tERROR: main repo to work with was not defined or did not exist" - else - writeHeader $mainRepoDir $antBuildFile - writeReposToRemove $antBuildFile - writeClosing $antBuildFile - fi -} - -function getReposToRemove () -{ - cDir="$1" - buildType=$2 - nRetain=$3 - buildDir=${remoteBase}/eclipse/downloads/drops4 - - if [[ ! -e "${cDir}" ]] - then - echo -e "\n\tERROR: expected directory did not exist" >&2 - echo -e "\t\t${cDir}" >&2 - reposToRemove=() - return 1 - else - echo -e "\n\tDEBUG: working with directory ${cDir}" - # for "repo names" we want only the last segment of the directory, so use -printf %f. The %C@ is seconds since the beginning of time, for sorting. - # Some caution is needed here. Seems on eclipse.org "atime" is the one that reflects "when created", - # whereas ctime and mtime are all identical, in every directory?! Turns out, mine is that - # say too. Apparently p2 "touches" every directory, for some reason. Perhaps only in the "atomic" case? - # But, atime can vary from system to system, depending .. some systems do update, when accessed? - sortedallOldRepos=( $(find ${cDir} -maxdepth 1 -type d -name "${buildType}*" -printf "%C@ %f\n" | sort | cut -d\ -f2 ) ) - #nOldRepos=${#sortedallOldRepos[@]} - # all builds "find" command should match above, except for age related (and printf) arguments - nbuilds=$( find ${cDir} -maxdepth 1 -type d -name "${buildType}*" | wc -l ) - echo -e "\tNumber of repos before cleaning: $nbuilds" - #echo -e "\tNumber of old repos ${nOldRepos}" - echo -e "\tDEBUG contents of sortedallOldRepos array" - for item in "${sortedallOldRepos[@]}" - do - echo -e "\t${item}" - done - #totalMinusOld=$(( nbuilds - nOldRepos )) - #echo -e "\tDEBUG: total minus old: $totalMinusOld" - - #remove unstable builds from the list - if [[ "$buildType" == "I" ]] - then - stableBuildRepos=() - for i in "${sortedallOldRepos[@]}" - do - if [[ ! -f ${buildDir}/${i}/buildUnstable ]] - then - stableBuildRepos+=(${i}) - fi - done - echo "Stable Builds" - for item in "${stableBuildRepos[@]}" - do - echo -e "\t${item}" - done - sortedallOldRepos=("${stableBuildRepos[@]}") - nbuilds=${#sortedallOldRepos[@]} - stableBuildRepos=() - fi - - echo -e "\tDEBUG contents of sortedallOldRepos array after removing unstable builds" - for item in "${sortedallOldRepos[@]}" - do - echo -e "\t${item}" - done - - if [[ $nbuilds -gt $nRetain ]] - then - # remove all old ones, except for nRetain - nToRemove=$(( nbuilds - nRetain )) - echo -e "\tDEBUG: nToRemove: $nToRemove" - #remove all except newest nRetain (if more than nRetain) - if [[ ${nToRemove} -gt 0 ]] - then - echo -e "\tDEBUG: number of old repos to remove found to be ${nToRemove}" - reposToRemove=("${sortedallOldRepos[@]:0:$nToRemove}") - else - echo -e "\tDEBUG: number of old repos to remove found to be ${nToRemove} so we will remove none" - reposToRemove=() - fi - fi - fi -} - -function cleanRepo () -{ - eclipseRepo=$1 - buildType=$2 - nRetain=$3 - dryRun=$4 - # Changed to "hard coded" location of where to expect on Hudson. - eclipseexe=${workspace}/eclipse/eclipse - if [[ ! -x ${eclipseexe} ]] - then - echo -e "\n\tERROR: expected eclipse location not found, or not executable" - echo -e "\t${eclipseexe}" - exit 1 - fi - javaexe=/opt/public/common/java/openjdk/jdk-21_x64-latest/bin/java - if [[ ! -x ${javaexe} ]] - then - echo -e "\n\tERROR: expected java location not found, or not executable" - echo -e "\t${javaexe}" - exit 1 - fi - - antBuildFile=${workspace}/cleanupRepoScript${buildType}.xml - antRunner=org.eclipse.ant.core.antRunner - - # To allow this cron job to work from hudson, or traditional crontab - devWorkspace=${workspace}/workspace-cleanup - - echo -e "\tDEBUG: Cleaning repository ${eclipseRepo} on $HOSTNAME on $(TZ="America/New_York" date ) " >&2 - getReposToRemove "${eclipseRepo}" $buildType $nRetain - RC=$? - if [[ $RC == 0 ]] - then - # be sure there are some to remove - nToRemove=${#reposToRemove[@]} - if [[ $nToRemove == 0 ]] - then - echo -e "\tfound no files to remove for current repo" - else - echo -e "\n\tfound $nToRemove so generating ant file" - generateCleanupXML "${eclipseRepo}" $antBuildFile - if [[ -z "${dryRun}" ]] - then - $eclipseexe -nosplash --launcher.suppressErrors -data "${devWorkspace}" -application ${antRunner} -f $antBuildFile -vm ${javaexe} - RC=$? - fi - if [[ $RC == 0 ]] - then - # we only clean N-build directories, others need to be manually cleaned - # after every milestone, or after every release - if [[ $buildType == "N" ]] - then - for file in "${reposToRemove[@]}" - do - echo -e "\tDEBUG: directories to remove: ${eclipseRepo}/${file}" - if [[ -z "${dryRun}" ]] - then - rm -rf ${eclipseRepo}/${file} - fi - done - else - echo -e "\n\tReminder: only composite cleaned. For $buildType builds must cleanup simple repos ever milestone or release". - fi - fi - if [[ -n "${dryRun}" ]] - then - echo "Since dryrun printing $antBuildFile" - cat $antBuildFile - fi - fi - fi -} - - -workspace=$1 -remoteBase="/home/data/httpd/download.eclipse.org" - -eclipseIRepo="${remoteBase}/eclipse/updates/4.37-I-builds" -eclipseYRepo="${remoteBase}/eclipse/updates/4.36-I-builds" -eclipseYRepo="${remoteBase}/eclipse/updates/4.37-Y-builds" -eclipsePRepo="${remoteBase}/eclipse/updates/4.36-Y-builds" -eclipseBuildTools="${remoteBase}/eclipse/updates/buildtools" - -doDryrun= -# global -declare -a reposToRemove=() -cleanRepo $eclipseIRepo I 2 $doDryrun -declare -a reposToRemove=() -cleanRepo $eclipseYRepo Y 2 $doDryrun -declare -a reposToRemove=() -cleanRepo $eclipsePRepo P 2 $doDryrun -declare -a reposToRemove=() -cleanRepo $eclipseBuildTools I 2 $doDryrun - -unset reposToRemove diff --git a/cje-production/mbscripts/mb620_promoteUpdateSite.sh b/cje-production/mbscripts/mb620_promoteUpdateSite.sh index 2edf66b827b..83424c41926 100755 --- a/cje-production/mbscripts/mb620_promoteUpdateSite.sh +++ b/cje-production/mbscripts/mb620_promoteUpdateSite.sh @@ -22,6 +22,7 @@ fi source $CJE_ROOT/scripts/common-functions.shsource source $1 +compositeTargetSize=3 epUpdateDir=/home/data/httpd/download.eclipse.org/eclipse/updates dropsPath=${epUpdateDir}/${STREAMMajor}.${STREAMMinor}-${BUILD_TYPE}-builds latestRelDir=/home/data/httpd/download.eclipse.org/eclipse/downloads/drops4 @@ -48,7 +49,35 @@ epRelDir=$(ssh genie.releng@projects-storage.eclipse.org ls -d --format=single-c ssh genie.releng@projects-storage.eclipse.org tar -C ${workspace} -xzf ${epRelDir}/eclipse-platform-*-linux-gtk-x86_64.tar.gz #get requisite tools -ssh genie.releng@projects-storage.eclipse.org wget -O ${workspace}/addToComposite.xml https://download.eclipse.org/eclipse/relengScripts/cje-production/scripts/addToComposite.xml +# Enhance the addToComposite ANT script to remove the oldest/earliest children to ensure the target-size is not exceeded +scp genie.releng@projects-storage.eclipse.org:${dropsPath}/compositeArtifacts.jar compositeArtifacts.jar +#Unzip compositeArtifacts.xml and read the current children from it +currentChildren=$(unzip -p compositeArtifacts.jar compositeArtifacts.xml |\ + xmllint - --xpath '/repository/children/child/@location' |\ + sed --expression 's|location="||g' --expression 's|"||g') +rm compositeArtifacts.jar +childrenArray=(${currentChildren}) +echo "Current children of composite repository: ${childrenArray[@]}" + +addToComposite_xml=$(curl https://download.eclipse.org/eclipse/relengScripts/cje-production/scripts/addToComposite.xml) +extraTasksMarker='' + +# One more child is about to be added to the composite. +# Remove those children from the beginning of the list if the target-size would be exceeded. +removalCount=$(( ${#childrenArray[@]} + 1 - compositeTargetSize )) +if [ "${removalCount}" -gt 0 ]; then + childrenToRemove="${childrenArray[@]:0:${removalCount}}" + echo "Remove from composite repsitory: ${childrenToRemove}" + extraAntTask='' + for child in ${childrenToRemove}; do + extraAntTask+="\\n " + done + addToComposite_xml=$(cat <<<"${addToComposite_xml}" | sed --expression "s|${extraTasksMarker}|${extraAntTask}|g") +else + echo "Composite p2-repository contains only ${#childrenArray[@]} children and adding one will not exceed its target size of ${compositeTargetSize}: ${dropsPath}" +fi + +ssh genie.releng@projects-storage.eclipse.org "cat<<<'${addToComposite_xml}' > ${workspace}/addToComposite.xml" #triggering ant runner baseBuilderDir=${workspace}/eclipse diff --git a/cje-production/scripts/addToComposite.xml b/cje-production/scripts/addToComposite.xml index 0394c553d01..968f0e2c6ef 100644 --- a/cje-production/scripts/addToComposite.xml +++ b/cje-production/scripts/addToComposite.xml @@ -1,5 +1,4 @@ - @@ -16,7 +15,7 @@ + -