diff --git a/JenkinsJobs/Cleanup/pruneDailyRepos.groovy b/JenkinsJobs/Cleanup/pruneDailyRepos.groovy deleted file mode 100644 index 6df2bb72685..00000000000 --- a/JenkinsJobs/Cleanup/pruneDailyRepos.groovy +++ /dev/null @@ -1,70 +0,0 @@ -job('Cleanup/pruneDailyRepos'){ - displayName('Daily Repo Pruner') - description(''' -This job runs several types of "cleanup" on the build machine and downloads server to remove old builds and other left overs from old build. -It acts as a simple cron job, currently running at 16:00 every day, to execute -.../sdk/cleaners/dailyCleanBuildMachine.sh -and other such scripts. - ''') - - logRotator { - daysToKeep(10) - numToKeep(5) - } - - jdk('temurin-jdk21-latest') - - label('basic') - - triggers { - cron(''' -0 5 * * * -0 17 * * * - ''') - } - - wrappers { //adds pre/post actions - timestamps() - preBuildCleanup() - sshAgent('projects-storage.eclipse.org-bot-ssh') - timeout { - absolute(30) - } - } - - steps { - shell(''' -#!/bin/bash -x - -epDownloadDir=/home/data/httpd/download.eclipse.org/eclipse -dropsPath=${epDownloadDir}/downloads/drops4 -p2RepoPath=${epDownloadDir}/updates -buildDir=${dropsPath}/${buildId} - -workingDir=${epDownloadDir}/workingDir - -workspace=${workingDir}/${JOB_NAME}-${BUILD_NUMBER} - -ssh genie.releng@projects-storage.eclipse.org rm -rf ${workingDir}/${JOB_NAME}* - -ssh genie.releng@projects-storage.eclipse.org mkdir -p ${workspace} -ssh genie.releng@projects-storage.eclipse.org cd ${workspace} - -#get latest Eclipse platform product -epRelDir=$(ssh genie.releng@projects-storage.eclipse.org ls -d --format=single-column ${dropsPath}/R-*|sort|tail -1) -ssh genie.releng@projects-storage.eclipse.org tar -C ${workspace} -xzf ${epRelDir}/eclipse-platform-*-linux-gtk-x86_64.tar.gz - -ssh genie.releng@projects-storage.eclipse.org wget -O ${workspace}/cleanupNightlyRepo.sh https://raw.githubusercontent.com/eclipse-platform/eclipse.platform.releng.aggregator/master/cje-production/cleaners/cleanupNightlyRepo.sh - -ssh genie.releng@projects-storage.eclipse.org bash -x ${workspace}/cleanupNightlyRepo.sh ${workspace} - -ssh genie.releng@projects-storage.eclipse.org rm -rf ${workingDir}/${JOB_NAME}* - ''') - } - - publishers { - extendedEmail { - recipientList("rahul.mohanan@ibm.com") - } - } -} diff --git a/cje-production/cleaners/cleanupNightlyRepo.sh b/cje-production/cleaners/cleanupNightlyRepo.sh deleted file mode 100755 index b52d38958f7..00000000000 --- a/cje-production/cleaners/cleanupNightlyRepo.sh +++ /dev/null @@ -1,242 +0,0 @@ -#!/bin/bash - -#******************************************************************************* -# Copyright (c) 2025 IBM Corporation and others. -# -# This program and the accompanying materials -# are made available under the terms of the Eclipse Public License 2.0 -# which accompanies this distribution, and is available at -# https://www.eclipse.org/legal/epl-2.0/ -# -# SPDX-License-Identifier: EPL-2.0 -# -# Contributors: -# IBM Corporation - initial API and implementation -#******************************************************************************* - -function writeHeader () -{ - compositeRepoDir="$1" - antBuildFile=$2 - if [[ -z "${compositeRepoDir}" ]] - then - echo -e "\n\tWARNING: compositeRepoDir not passed to writeHeader function as expected. But will continue with variable for later use?" - compositeRepoDir="\$\{compositeRepoDir\}" - fi - echo -e "" > $antBuildFile - echo -e "> $antBuildFile - echo -e " basedir=\".\"" >>$antBuildFile - echo -e " default=\"cleanup\">" >>$antBuildFile - echo -e " " >>$antBuildFile - echo -e " " >>$antBuildFile - echo -e " " >>$antBuildFile - echo -e " " >> $antBuildFile -} - -function writeReposToRemove () -{ - antBuildFile=$1 - for repo in "${reposToRemove[@]}" - do - echo " " >> $antBuildFile - done - -} - -function writeClosing () -{ - antBuildFile=$1 - echo -e " " >> $antBuildFile - echo -e " " >> $antBuildFile - echo -e " " >> $antBuildFile - echo -e "" >> $antBuildFile -} - -function generateCleanupXML () -{ - mainRepoDir=$1 - antBuildFile=$2 - if [[ -z "${mainRepoDir}" || ! -e "${mainRepoDir}" ]] - then - echo -e "\n\tERROR: main repo to work with was not defined or did not exist" - else - writeHeader $mainRepoDir $antBuildFile - writeReposToRemove $antBuildFile - writeClosing $antBuildFile - fi -} - -function getReposToRemove () -{ - cDir="$1" - buildType=$2 - nRetain=$3 - buildDir=${remoteBase}/eclipse/downloads/drops4 - - if [[ ! -e "${cDir}" ]] - then - echo -e "\n\tERROR: expected directory did not exist" >&2 - echo -e "\t\t${cDir}" >&2 - reposToRemove=() - return 1 - else - echo -e "\n\tDEBUG: working with directory ${cDir}" - # for "repo names" we want only the last segment of the directory, so use -printf %f. The %C@ is seconds since the beginning of time, for sorting. - # Some caution is needed here. Seems on eclipse.org "atime" is the one that reflects "when created", - # whereas ctime and mtime are all identical, in every directory?! Turns out, mine is that - # say too. Apparently p2 "touches" every directory, for some reason. Perhaps only in the "atomic" case? - # But, atime can vary from system to system, depending .. some systems do update, when accessed? - sortedallOldRepos=( $(find ${cDir} -maxdepth 1 -type d -name "${buildType}*" -printf "%C@ %f\n" | sort | cut -d\ -f2 ) ) - #nOldRepos=${#sortedallOldRepos[@]} - # all builds "find" command should match above, except for age related (and printf) arguments - nbuilds=$( find ${cDir} -maxdepth 1 -type d -name "${buildType}*" | wc -l ) - echo -e "\tNumber of repos before cleaning: $nbuilds" - #echo -e "\tNumber of old repos ${nOldRepos}" - echo -e "\tDEBUG contents of sortedallOldRepos array" - for item in "${sortedallOldRepos[@]}" - do - echo -e "\t${item}" - done - #totalMinusOld=$(( nbuilds - nOldRepos )) - #echo -e "\tDEBUG: total minus old: $totalMinusOld" - - #remove unstable builds from the list - if [[ "$buildType" == "I" ]] - then - stableBuildRepos=() - for i in "${sortedallOldRepos[@]}" - do - if [[ ! -f ${buildDir}/${i}/buildUnstable ]] - then - stableBuildRepos+=(${i}) - fi - done - echo "Stable Builds" - for item in "${stableBuildRepos[@]}" - do - echo -e "\t${item}" - done - sortedallOldRepos=("${stableBuildRepos[@]}") - nbuilds=${#sortedallOldRepos[@]} - stableBuildRepos=() - fi - - echo -e "\tDEBUG contents of sortedallOldRepos array after removing unstable builds" - for item in "${sortedallOldRepos[@]}" - do - echo -e "\t${item}" - done - - if [[ $nbuilds -gt $nRetain ]] - then - # remove all old ones, except for nRetain - nToRemove=$(( nbuilds - nRetain )) - echo -e "\tDEBUG: nToRemove: $nToRemove" - #remove all except newest nRetain (if more than nRetain) - if [[ ${nToRemove} -gt 0 ]] - then - echo -e "\tDEBUG: number of old repos to remove found to be ${nToRemove}" - reposToRemove=("${sortedallOldRepos[@]:0:$nToRemove}") - else - echo -e "\tDEBUG: number of old repos to remove found to be ${nToRemove} so we will remove none" - reposToRemove=() - fi - fi - fi -} - -function cleanRepo () -{ - eclipseRepo=$1 - buildType=$2 - nRetain=$3 - dryRun=$4 - # Changed to "hard coded" location of where to expect on Hudson. - eclipseexe=${workspace}/eclipse/eclipse - if [[ ! -x ${eclipseexe} ]] - then - echo -e "\n\tERROR: expected eclipse location not found, or not executable" - echo -e "\t${eclipseexe}" - exit 1 - fi - javaexe=/opt/public/common/java/openjdk/jdk-21_x64-latest/bin/java - if [[ ! -x ${javaexe} ]] - then - echo -e "\n\tERROR: expected java location not found, or not executable" - echo -e "\t${javaexe}" - exit 1 - fi - - antBuildFile=${workspace}/cleanupRepoScript${buildType}.xml - antRunner=org.eclipse.ant.core.antRunner - - # To allow this cron job to work from hudson, or traditional crontab - devWorkspace=${workspace}/workspace-cleanup - - echo -e "\tDEBUG: Cleaning repository ${eclipseRepo} on $HOSTNAME on $(TZ="America/New_York" date ) " >&2 - getReposToRemove "${eclipseRepo}" $buildType $nRetain - RC=$? - if [[ $RC == 0 ]] - then - # be sure there are some to remove - nToRemove=${#reposToRemove[@]} - if [[ $nToRemove == 0 ]] - then - echo -e "\tfound no files to remove for current repo" - else - echo -e "\n\tfound $nToRemove so generating ant file" - generateCleanupXML "${eclipseRepo}" $antBuildFile - if [[ -z "${dryRun}" ]] - then - $eclipseexe -nosplash --launcher.suppressErrors -data "${devWorkspace}" -application ${antRunner} -f $antBuildFile -vm ${javaexe} - RC=$? - fi - if [[ $RC == 0 ]] - then - # we only clean N-build directories, others need to be manually cleaned - # after every milestone, or after every release - if [[ $buildType == "N" ]] - then - for file in "${reposToRemove[@]}" - do - echo -e "\tDEBUG: directories to remove: ${eclipseRepo}/${file}" - if [[ -z "${dryRun}" ]] - then - rm -rf ${eclipseRepo}/${file} - fi - done - else - echo -e "\n\tReminder: only composite cleaned. For $buildType builds must cleanup simple repos ever milestone or release". - fi - fi - if [[ -n "${dryRun}" ]] - then - echo "Since dryrun printing $antBuildFile" - cat $antBuildFile - fi - fi - fi -} - - -workspace=$1 -remoteBase="/home/data/httpd/download.eclipse.org" - -eclipseIRepo="${remoteBase}/eclipse/updates/4.37-I-builds" -eclipseYRepo="${remoteBase}/eclipse/updates/4.36-I-builds" -eclipseYRepo="${remoteBase}/eclipse/updates/4.37-Y-builds" -eclipsePRepo="${remoteBase}/eclipse/updates/4.36-Y-builds" -eclipseBuildTools="${remoteBase}/eclipse/updates/buildtools" - -doDryrun= -# global -declare -a reposToRemove=() -cleanRepo $eclipseIRepo I 2 $doDryrun -declare -a reposToRemove=() -cleanRepo $eclipseYRepo Y 2 $doDryrun -declare -a reposToRemove=() -cleanRepo $eclipsePRepo P 2 $doDryrun -declare -a reposToRemove=() -cleanRepo $eclipseBuildTools I 2 $doDryrun - -unset reposToRemove diff --git a/cje-production/mbscripts/mb620_promoteUpdateSite.sh b/cje-production/mbscripts/mb620_promoteUpdateSite.sh index 2edf66b827b..83424c41926 100755 --- a/cje-production/mbscripts/mb620_promoteUpdateSite.sh +++ b/cje-production/mbscripts/mb620_promoteUpdateSite.sh @@ -22,6 +22,7 @@ fi source $CJE_ROOT/scripts/common-functions.shsource source $1 +compositeTargetSize=3 epUpdateDir=/home/data/httpd/download.eclipse.org/eclipse/updates dropsPath=${epUpdateDir}/${STREAMMajor}.${STREAMMinor}-${BUILD_TYPE}-builds latestRelDir=/home/data/httpd/download.eclipse.org/eclipse/downloads/drops4 @@ -48,7 +49,35 @@ epRelDir=$(ssh genie.releng@projects-storage.eclipse.org ls -d --format=single-c ssh genie.releng@projects-storage.eclipse.org tar -C ${workspace} -xzf ${epRelDir}/eclipse-platform-*-linux-gtk-x86_64.tar.gz #get requisite tools -ssh genie.releng@projects-storage.eclipse.org wget -O ${workspace}/addToComposite.xml https://download.eclipse.org/eclipse/relengScripts/cje-production/scripts/addToComposite.xml +# Enhance the addToComposite ANT script to remove the oldest/earliest children to ensure the target-size is not exceeded +scp genie.releng@projects-storage.eclipse.org:${dropsPath}/compositeArtifacts.jar compositeArtifacts.jar +#Unzip compositeArtifacts.xml and read the current children from it +currentChildren=$(unzip -p compositeArtifacts.jar compositeArtifacts.xml |\ + xmllint - --xpath '/repository/children/child/@location' |\ + sed --expression 's|location="||g' --expression 's|"||g') +rm compositeArtifacts.jar +childrenArray=(${currentChildren}) +echo "Current children of composite repository: ${childrenArray[@]}" + +addToComposite_xml=$(curl https://download.eclipse.org/eclipse/relengScripts/cje-production/scripts/addToComposite.xml) +extraTasksMarker='' + +# One more child is about to be added to the composite. +# Remove those children from the beginning of the list if the target-size would be exceeded. +removalCount=$(( ${#childrenArray[@]} + 1 - compositeTargetSize )) +if [ "${removalCount}" -gt 0 ]; then + childrenToRemove="${childrenArray[@]:0:${removalCount}}" + echo "Remove from composite repsitory: ${childrenToRemove}" + extraAntTask='' + for child in ${childrenToRemove}; do + extraAntTask+="\\n " + done + addToComposite_xml=$(cat <<<"${addToComposite_xml}" | sed --expression "s|${extraTasksMarker}|${extraAntTask}|g") +else + echo "Composite p2-repository contains only ${#childrenArray[@]} children and adding one will not exceed its target size of ${compositeTargetSize}: ${dropsPath}" +fi + +ssh genie.releng@projects-storage.eclipse.org "cat<<<'${addToComposite_xml}' > ${workspace}/addToComposite.xml" #triggering ant runner baseBuilderDir=${workspace}/eclipse diff --git a/cje-production/scripts/addToComposite.xml b/cje-production/scripts/addToComposite.xml index 0394c553d01..968f0e2c6ef 100644 --- a/cje-production/scripts/addToComposite.xml +++ b/cje-production/scripts/addToComposite.xml @@ -1,5 +1,4 @@ - @@ -16,7 +15,7 @@ + -