|
| 1 | + |
| 2 | +def int EQUINOX_RETENTION_COUNT = 3 // Keep the three most recent builds |
| 3 | +def int ECLIPSE_I_BUILD_RETENTION_DAYS = 5 // Minimal number of days for which a drop should be retained |
| 4 | +def int ECLIPSE_I_BUILD_RETENTION_COUNT = 4 // Minimal number of drops to retain regardless of their age |
| 5 | +def int ECLIPSE_Y_BUILD_RETENTION_COUNT = 9 // Number of drops retained for Y-builds (corresponds to builds of three consecutive weeks) |
| 6 | + |
| 7 | +pipeline { |
| 8 | + options { |
| 9 | + skipDefaultCheckout() |
| 10 | + timestamps() |
| 11 | + timeout(time: 15, unit: 'MINUTES') |
| 12 | + buildDiscarder(logRotator(numToKeepStr:'5')) |
| 13 | + } |
| 14 | + agent { |
| 15 | + label 'basic' |
| 16 | + } |
| 17 | + environment { |
| 18 | + // Download Server locations (would very seldom change) |
| 19 | + EP_ROOT = '/home/data/httpd/download.eclipse.org' |
| 20 | + EP_ECLIPSE_DROPS = "${EP_ROOT}/eclipse/downloads/drops4" |
| 21 | + EP_EQUINOX_DROPS = "${EP_ROOT}/equinox/drops" |
| 22 | + |
| 23 | + |
| 24 | + } |
| 25 | + stages { |
| 26 | + stage('Checkout SCM') { |
| 27 | + steps { |
| 28 | + checkout scmGit(userRemoteConfigs: [[url: "${scm.userRemoteConfigs[0].url}"]], branches: [[name: "${scm.branches[0].name}"]], |
| 29 | + extensions: [cloneOption(depth: 1, shallow: true, noTags: true), sparseCheckout([ |
| 30 | + [path: 'JenkinsJobs/shared/utilities.groovy'], |
| 31 | + [path: 'cje-production/buildproperties.txt'], |
| 32 | + ])]) |
| 33 | + script { |
| 34 | + utilities = load "JenkinsJobs/shared/utilities.groovy" |
| 35 | + utilities.setDryRun(false) |
| 36 | + def buildProperties = readProperties(file: 'cje-production/buildproperties.txt') |
| 37 | + devVersionMajor = buildProperties.STREAMMajor.replace('"','').toInteger() // Remove surrounding quotes |
| 38 | + devVersionMinor = buildProperties.STREAMMinor.replace('"','').toInteger() // Remove surrounding quotes |
| 39 | + devVersionService = buildProperties.STREAMService.replace('"','').toInteger() // Remove surrounding quotes |
| 40 | + } |
| 41 | + } |
| 42 | + } |
| 43 | + stage('Clean up builds') { |
| 44 | + parallel { |
| 45 | + stage('Equinox') { |
| 46 | + steps { |
| 47 | + sshagent (['projects-storage.eclipse.org-bot-ssh']) { |
| 48 | + removeSurplusBuildDrops("${EP_EQUINOX_DROPS}", 'I', EQUINOX_RETENTION_COUNT) |
| 49 | + removeSurplusBuildDrops("${EP_EQUINOX_DROPS}", 'Y', EQUINOX_RETENTION_COUNT) |
| 50 | + } |
| 51 | + } |
| 52 | + } |
| 53 | + stage('Eclipse I-builds') { |
| 54 | + steps { |
| 55 | + sshagent (['projects-storage.eclipse.org-bot-ssh']) { |
| 56 | + removeOldBuildDropsOfCurrentStream("${EP_ECLIPSE_DROPS}", 'I', ECLIPSE_I_BUILD_RETENTION_DAYS, ECLIPSE_I_BUILD_RETENTION_COUNT) |
| 57 | + } |
| 58 | + } |
| 59 | + } |
| 60 | + stage('Eclipse Y-builds') { |
| 61 | + steps { |
| 62 | + sshagent (['projects-storage.eclipse.org-bot-ssh']) { |
| 63 | + removeSurplusBuildDrops("${EP_ECLIPSE_DROPS}", 'Y', ECLIPSE_Y_BUILD_RETENTION_COUNT) |
| 64 | + } |
| 65 | + } |
| 66 | + } |
| 67 | + } |
| 68 | + post { |
| 69 | + success { |
| 70 | + build job: 'Releng/updateIndex', wait: false |
| 71 | + } |
| 72 | + } |
| 73 | + } |
| 74 | + } |
| 75 | +} |
| 76 | + |
| 77 | +@groovy.transform.Field |
| 78 | +def utilities = null |
| 79 | +@groovy.transform.Field |
| 80 | +def int devVersionMajor = null |
| 81 | +@groovy.transform.Field |
| 82 | +def int devVersionMinor = null |
| 83 | +@groovy.transform.Field |
| 84 | +def int devVersionService = null |
| 85 | + |
| 86 | +def removeSurplusBuildDrops(String remoteDirectory, String buildType, int retentionCount) { |
| 87 | + def drops = utilities.listBuildDropDirectoriesOnRemote(remoteDirectory, "${buildType}*") // sorted in ascending order |
| 88 | + if (retentionCount < drops.size()) { |
| 89 | + utilities.removeDropsOnRemote(remoteDirectory, drops.subList(0, drops.size() - retentionCount)) |
| 90 | + } else { |
| 91 | + echo "Nothing to clean in ${remoteDirectory} with pattern '${buildType}*'. Found only ${drops}, not exceeding the threshold of ${retentionCount}." |
| 92 | + } |
| 93 | +} |
| 94 | + |
| 95 | +def removeOldBuildDropsOfCurrentStream(String remoteDirectory, String buildType, int retentionDays, int retentionCount) { |
| 96 | + // Among all builds older than minimal retention time, keep only the first stable build of each week (starting Monday) and delete all others. |
| 97 | + // But keep at least the minimal retention count of builds regardless of their age and all of those younger than the minimal retention time. |
| 98 | + def retentionThresholdDate = java.time.LocalDate.now().minusDays(retentionDays) |
| 99 | + def allBuilds = utilities.listBuildDropDirectoriesOnRemote(remoteDirectory, "${buildType}*", |
| 100 | + devVersionMajor, devVersionMinor, devVersionService).sort() // sort ascending to start with Mondays |
| 101 | + echo "Number of ${buildType}-builds before cleaning: ${allBuilds.size()}" |
| 102 | + def oldBuilds = allBuilds.findAll{ b -> parseDate(b) < retentionThresholdDate} // sorted ascendingly to start with Mondays |
| 103 | + // Keep the specified count of latest builds |
| 104 | + def toRetain = allBuilds.subList(Math.max(allBuilds.size() - retentionCount, 0), allBuilds.size()).toSet() |
| 105 | + echo "Old ${buildType}-Builds: ${oldBuilds}" |
| 106 | + echo "Minimally retained ${buildType}-Builds: ${toRetain}" |
| 107 | + def Set<Integer> retainedWeeks = [] |
| 108 | + def toRemove = [] |
| 109 | + for (oldBuild in oldBuilds) { |
| 110 | + if (toRetain.contains(oldBuild)) { |
| 111 | + echo "Not removed (since one of the ${retentionCount} newest builds, even though old): ${oldBuild}" |
| 112 | + continue; |
| 113 | + } |
| 114 | + def weekOfYear = parseDate(oldBuild).get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR) |
| 115 | + if (!retainedWeeks.contains(weekOfYear)) { |
| 116 | + def isUnstable = sh(script: """$SSH "if [ -f '${remoteDirectory}/${oldBuild}/buildUnstable' ]; then echo 'true'; fi" """, returnStdout: true).toBoolean() |
| 117 | + if (!isUnstable) { |
| 118 | + retainedWeeks.add(weekOfYear) // Retain the first stable build of a week |
| 119 | + echo "Retain stable build ${oldBuild} for week ${weekOfYear}" |
| 120 | + continue; |
| 121 | + } else { |
| 122 | + echo "Ignore unstable build ${oldBuild} to be retained for week ${weekOfYear}" |
| 123 | + } |
| 124 | + } |
| 125 | + toRemove.add(oldBuild) |
| 126 | + } |
| 127 | + if (!toRemove.isEmpty()) { |
| 128 | + utilities.removeDropsOnRemote(remoteDirectory, toRemove) |
| 129 | + } else { |
| 130 | + echo "Nothing to clean in ${remoteDirectory} with pattern '${buildType}*' and retention of at least ${retentionDays} days and ${retentionCount} builds." |
| 131 | + } |
| 132 | +} |
| 133 | + |
| 134 | +@NonCPS |
| 135 | +def parseDate(String iBuildId) { |
| 136 | + def idMatcher = iBuildId =~ /I(?<date>\d{8})-(?<time>\d{4})/ |
| 137 | + if (!idMatcher.matches()) { |
| 138 | + error "buildID: ${iBuildId}, does not match the expected pattern." |
| 139 | + } |
| 140 | + def date = idMatcher.group('date') |
| 141 | + return java.time.LocalDate.of(date.substring(0, 4).toInteger(), date.substring(4, 6).toInteger(), date.substring(6, 8).toInteger()) |
| 142 | +} |
0 commit comments