From ce59fe8fdce0c818eeda0b326d68dae62859f61a Mon Sep 17 00:00:00 2001 From: Christian Beikov Date: Tue, 22 Oct 2024 16:21:25 +0200 Subject: [PATCH 1/4] Don't use the build cache during releases So that we're extra sure we're not relying on cache left by a previous Gradle execution. (cherry picked from commit 543e095cf70396256ebbf2dd9e9d07f80304bebd) --- ci/release/Jenkinsfile | 11 +++++++++-- gradle/gradle-enterprise.gradle | 11 ++++++++++- settings.gradle | 2 +- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/ci/release/Jenkinsfile b/ci/release/Jenkinsfile index 026237cec8be..6b6ca9089f5c 100644 --- a/ci/release/Jenkinsfile +++ b/ci/release/Jenkinsfile @@ -135,7 +135,10 @@ pipeline { // update changelog from JIRA // tags the version // changes the version to the provided development version - withEnv(["BRANCH=${env.GIT_BRANCH}"]) { + withEnv([ + "BRANCH=${env.GIT_BRANCH}", + "DISABLE_REMOTE_GRADLE_CACHE=true" + ]) { sh ".release/scripts/prepare-release.sh ${env.PROJECT} ${env.RELEASE_VERSION} ${env.DEVELOPMENT_VERSION}" } } @@ -161,7 +164,11 @@ pipeline { sshagent(['ed25519.Hibernate-CI.github.com', 'hibernate.filemgmt.jboss.org', 'hibernate-ci.frs.sourceforge.net']) { // performs documentation upload and Sonatype release // push to github - sh ".release/scripts/publish.sh ${env.SCRIPT_OPTIONS} ${env.PROJECT} ${env.RELEASE_VERSION} ${env.DEVELOPMENT_VERSION} ${env.GIT_BRANCH}" + withEnv([ + "DISABLE_REMOTE_GRADLE_CACHE=true" + ]) { + sh ".release/scripts/publish.sh ${env.SCRIPT_OPTIONS} ${env.PROJECT} ${env.RELEASE_VERSION} ${env.DEVELOPMENT_VERSION} ${env.GIT_BRANCH}" + } } } } diff --git a/gradle/gradle-enterprise.gradle b/gradle/gradle-enterprise.gradle index 700b0618af9e..107b4d22fd97 100644 --- a/gradle/gradle-enterprise.gradle +++ b/gradle/gradle-enterprise.gradle @@ -11,7 +11,8 @@ ext { isCiEnvironment = isJenkins() || isGitHubActions() || isGenericCi() - populateRemoteBuildCache = getSetting( "POPULATE_REMOTE" ).isPresent() + populateRemoteBuildCache = isEnabled( "POPULATE_REMOTE" ) + useRemoteCache = !isEnabled( "DISABLE_REMOTE_GRADLE_CACHE" ) } private static boolean isJenkins() { @@ -36,6 +37,14 @@ static java.util.Optional getSetting(String name) { return java.util.Optional.ofNullable(sysProp); } +static boolean isEnabled(String setting) { + if ( System.getenv().hasProperty( setting ) ) { + return true + } + + return System.hasProperty( setting ) +} + gradleEnterprise { server = 'https://ge.hibernate.org' diff --git a/settings.gradle b/settings.gradle index 4675fded283b..0028111bb190 100644 --- a/settings.gradle +++ b/settings.gradle @@ -279,7 +279,7 @@ buildCache { enabled = !settings.ext.isCiEnvironment } remote(HttpBuildCache) { - enabled = true + enabled = settings.ext.useRemoteCache push = settings.ext.populateRemoteBuildCache url = 'https://ge.hibernate.org/cache/' } From 38f364c89eebd6c433e4057fe0a60968498536b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yoann=20Rodi=C3=A8re?= Date: Mon, 28 Oct 2024 17:19:55 +0100 Subject: [PATCH 2/4] Sanitize user home on GitHub Action job start on Atlas (cherry picked from commit 4e8eb91562b75fbcc6ecdaaa3efe62b724f71aa4) --- .github/ci-prerequisites-atlas.sh | 6 ++++++ .github/workflows/atlas.yml | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100755 .github/ci-prerequisites-atlas.sh diff --git a/.github/ci-prerequisites-atlas.sh b/.github/ci-prerequisites-atlas.sh new file mode 100755 index 000000000000..b0a16f22b1e8 --- /dev/null +++ b/.github/ci-prerequisites-atlas.sh @@ -0,0 +1,6 @@ +# Reclaims disk space and sanitizes user home on Atlas infrastructure + +# We use the GitHub cache for the relevant parts of these directories. +# Also, we do not want to keep things like ~/.gradle/build-scan-data. +rm -rf ~/.gradle/ +rm -rf ~/.m2/ diff --git a/.github/workflows/atlas.yml b/.github/workflows/atlas.yml index 65eeea82ac1d..edf84fe55849 100644 --- a/.github/workflows/atlas.yml +++ b/.github/workflows/atlas.yml @@ -41,8 +41,8 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false - - name: Reclaim Disk Space - run: .github/ci-prerequisites.sh + - name: Reclaim disk space and sanitize user home + run: .github/ci-prerequisites-atlas.sh - name: Start database env: RDBMS: ${{ matrix.rdbms }} From f5b28e671e8014c020c09c770f549a81652d7b6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yoann=20Rodi=C3=A8re?= Date: Fri, 25 Oct 2024 11:28:29 +0200 Subject: [PATCH 3/4] Improve build cache strategy Ripped off from Quarkus. Here's how it will work: 1. We create a cache entry on push only. Pull requests only restore it. 2. We create a new cache entry every day, prefixed with something like 2024-10-25. 3. When restoring the cache, we try the entry for the day first (2024-10-25) and default to the one for the month (2024-10-*). Critically, this means we will build each day's cache based on the previous day's cache. 4. Atlas infra uses its own, separate cache entries. Co-Authored-By: Guillaume Smet (cherry picked from commit 5eaaff2e6367fd4f1071a90eba2e274c800629cd) --- .github/workflows/atlas.yml | 48 ++++++++++++++++++++----- .github/workflows/contributor-build.yml | 48 ++++++++++++++++++++----- 2 files changed, 78 insertions(+), 18 deletions(-) diff --git a/.github/workflows/atlas.yml b/.github/workflows/atlas.yml index edf84fe55849..db71e178eb57 100644 --- a/.github/workflows/atlas.yml +++ b/.github/workflows/atlas.yml @@ -53,20 +53,50 @@ jobs: with: distribution: 'graalvm' java-version: '21' - - name: Get year/month for cache key - id: get-date - run: echo "yearmonth=$(/bin/date -u "+%Y-%m")" >> $GITHUB_OUTPUT - shell: bash - - name: Cache Maven local repository - uses: actions/cache@v3 + + - name: Generate cache key + id: cache-key + run: | + CURRENT_BRANCH="${{ github.repository != 'hibernate/hibernate-orm' && 'fork' || github.base_ref || github.ref_name }}" + CURRENT_MONTH=$(/bin/date -u "+%Y-%m") + CURRENT_DAY=$(/bin/date -u "+%d") + ROOT_CACHE_KEY="buildtool-cache-atlas" + echo "buildtool-monthly-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}" >> $GITHUB_OUTPUT + echo "buildtool-monthly-branch-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}-${CURRENT_BRANCH}" >> $GITHUB_OUTPUT + echo "buildtool-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}-${CURRENT_BRANCH}-${CURRENT_DAY}" >> $GITHUB_OUTPUT + - name: Cache Maven/Gradle Local Caches id: cache-maven + uses: actions/cache@v4 + # if it's not a pull request, we restore and save the cache + if: github.event_name != 'pull_request' with: path: | - ~/.m2/repository + ~/.m2/repository/ + ~/.m2/wrapper/ ~/.gradle/caches/ ~/.gradle/wrapper/ - # refresh cache every month to avoid unlimited growth - key: maven-localrepo-${{ steps.get-date.outputs.yearmonth }} + # A new cache will be stored daily. After that first store of the day, cache save actions will fail because the cache is immutable but it's not a problem. + # The whole cache is dropped monthly to prevent unlimited growth. + # The cache is per branch but in case we don't find a branch for a given branch, we will get a cache from another branch. + key: ${{ steps.cache-key.outputs.buildtool-cache-key }} + restore-keys: | + ${{ steps.cache-key.outputs.buildtool-monthly-branch-cache-key }}- + ${{ steps.cache-key.outputs.buildtool-monthly-cache-key }}- + - name: Restore Maven/Gradle Local Caches + uses: actions/cache/restore@v3 + # if it a pull request, we restore the cache but we don't save it + if: github.event_name == 'pull_request' + with: + path: | + ~/.m2/repository/ + ~/.m2/wrapper/ + ~/.gradle/caches/ + ~/.gradle/wrapper/ + key: ${{ steps.cache-key.outputs.buildtool-cache-key }} + restore-keys: | + ${{ steps.cache-key.outputs.buildtool-monthly-branch-cache-key }}- + ${{ steps.cache-key.outputs.buildtool-monthly-cache-key }}- + - name: Run build script env: RDBMS: ${{ matrix.rdbms }} diff --git a/.github/workflows/contributor-build.yml b/.github/workflows/contributor-build.yml index c626cb663895..78949914d1b9 100644 --- a/.github/workflows/contributor-build.yml +++ b/.github/workflows/contributor-build.yml @@ -62,20 +62,50 @@ jobs: with: distribution: 'temurin' java-version: '11' - - name: Get year/month for cache key - id: get-date - run: echo "yearmonth=$(/bin/date -u "+%Y-%m")" >> $GITHUB_OUTPUT - shell: bash - - name: Cache Maven local repository - uses: actions/cache@v3 + + - name: Generate cache key + id: cache-key + run: | + CURRENT_BRANCH="${{ github.repository != 'hibernate/hibernate-orm' && 'fork' || github.base_ref || github.ref_name }}" + CURRENT_MONTH=$(/bin/date -u "+%Y-%m") + CURRENT_DAY=$(/bin/date -u "+%d") + ROOT_CACHE_KEY="buildtool-cache" + echo "buildtool-monthly-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}" >> $GITHUB_OUTPUT + echo "buildtool-monthly-branch-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}-${CURRENT_BRANCH}" >> $GITHUB_OUTPUT + echo "buildtool-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}-${CURRENT_BRANCH}-${CURRENT_DAY}" >> $GITHUB_OUTPUT + - name: Cache Maven/Gradle Local Caches id: cache-maven + uses: actions/cache@v4 + # if it's not a pull request, we restore and save the cache + if: github.event_name != 'pull_request' with: path: | - ~/.m2/repository + ~/.m2/repository/ + ~/.m2/wrapper/ ~/.gradle/caches/ ~/.gradle/wrapper/ - # refresh cache every month to avoid unlimited growth - key: maven-localrepo-${{ steps.get-date.outputs.yearmonth }} + # A new cache will be stored daily. After that first store of the day, cache save actions will fail because the cache is immutable but it's not a problem. + # The whole cache is dropped monthly to prevent unlimited growth. + # The cache is per branch but in case we don't find a branch for a given branch, we will get a cache from another branch. + key: ${{ steps.cache-key.outputs.buildtool-cache-key }} + restore-keys: | + ${{ steps.cache-key.outputs.buildtool-monthly-branch-cache-key }}- + ${{ steps.cache-key.outputs.buildtool-monthly-cache-key }}- + - name: Restore Maven/Gradle Local Caches + uses: actions/cache/restore@v3 + # if it a pull request, we restore the cache but we don't save it + if: github.event_name == 'pull_request' + with: + path: | + ~/.m2/repository/ + ~/.m2/wrapper/ + ~/.gradle/caches/ + ~/.gradle/wrapper/ + key: ${{ steps.cache-key.outputs.buildtool-cache-key }} + restore-keys: | + ${{ steps.cache-key.outputs.buildtool-monthly-branch-cache-key }}- + ${{ steps.cache-key.outputs.buildtool-monthly-cache-key }}- + - name: Run build script env: RDBMS: ${{ matrix.rdbms }} From e316da7bf82f84635644b190017a9ea24700c5cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yoann=20Rodi=C3=A8re?= Date: Fri, 25 Oct 2024 12:27:06 +0200 Subject: [PATCH 4/4] Do not include the whole Gradle cache in GH Actions caches We don't need the whole cache, since most (all?) of it is stored on Develocity already. Including ./gradle/caches/modules-2 is enough, because that's where Maven dependencies are stored, and *those* are not stored on Develocity. (cherry picked from commit 013bb46470f97285ae57190c9ce5201f31105a21) --- .github/workflows/atlas.yml | 8 ++++---- .github/workflows/contributor-build.yml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/atlas.yml b/.github/workflows/atlas.yml index db71e178eb57..83a59e0f5eea 100644 --- a/.github/workflows/atlas.yml +++ b/.github/workflows/atlas.yml @@ -64,7 +64,7 @@ jobs: echo "buildtool-monthly-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}" >> $GITHUB_OUTPUT echo "buildtool-monthly-branch-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}-${CURRENT_BRANCH}" >> $GITHUB_OUTPUT echo "buildtool-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}-${CURRENT_BRANCH}-${CURRENT_DAY}" >> $GITHUB_OUTPUT - - name: Cache Maven/Gradle Local Caches + - name: Cache Maven/Gradle Dependency/Dist Caches id: cache-maven uses: actions/cache@v4 # if it's not a pull request, we restore and save the cache @@ -73,7 +73,7 @@ jobs: path: | ~/.m2/repository/ ~/.m2/wrapper/ - ~/.gradle/caches/ + ~/.gradle/caches/modules-2 ~/.gradle/wrapper/ # A new cache will be stored daily. After that first store of the day, cache save actions will fail because the cache is immutable but it's not a problem. # The whole cache is dropped monthly to prevent unlimited growth. @@ -82,7 +82,7 @@ jobs: restore-keys: | ${{ steps.cache-key.outputs.buildtool-monthly-branch-cache-key }}- ${{ steps.cache-key.outputs.buildtool-monthly-cache-key }}- - - name: Restore Maven/Gradle Local Caches + - name: Restore Maven/Gradle Dependency/Dist Caches uses: actions/cache/restore@v3 # if it a pull request, we restore the cache but we don't save it if: github.event_name == 'pull_request' @@ -90,7 +90,7 @@ jobs: path: | ~/.m2/repository/ ~/.m2/wrapper/ - ~/.gradle/caches/ + ~/.gradle/caches/modules-2 ~/.gradle/wrapper/ key: ${{ steps.cache-key.outputs.buildtool-cache-key }} restore-keys: | diff --git a/.github/workflows/contributor-build.yml b/.github/workflows/contributor-build.yml index 78949914d1b9..2d06f42c0fd0 100644 --- a/.github/workflows/contributor-build.yml +++ b/.github/workflows/contributor-build.yml @@ -73,7 +73,7 @@ jobs: echo "buildtool-monthly-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}" >> $GITHUB_OUTPUT echo "buildtool-monthly-branch-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}-${CURRENT_BRANCH}" >> $GITHUB_OUTPUT echo "buildtool-cache-key=${ROOT_CACHE_KEY}-${CURRENT_MONTH}-${CURRENT_BRANCH}-${CURRENT_DAY}" >> $GITHUB_OUTPUT - - name: Cache Maven/Gradle Local Caches + - name: Cache Maven/Gradle Dependency/Dist Caches id: cache-maven uses: actions/cache@v4 # if it's not a pull request, we restore and save the cache @@ -82,7 +82,7 @@ jobs: path: | ~/.m2/repository/ ~/.m2/wrapper/ - ~/.gradle/caches/ + ~/.gradle/caches/modules-2 ~/.gradle/wrapper/ # A new cache will be stored daily. After that first store of the day, cache save actions will fail because the cache is immutable but it's not a problem. # The whole cache is dropped monthly to prevent unlimited growth. @@ -91,7 +91,7 @@ jobs: restore-keys: | ${{ steps.cache-key.outputs.buildtool-monthly-branch-cache-key }}- ${{ steps.cache-key.outputs.buildtool-monthly-cache-key }}- - - name: Restore Maven/Gradle Local Caches + - name: Restore Maven/Gradle Dependency/Dist Caches uses: actions/cache/restore@v3 # if it a pull request, we restore the cache but we don't save it if: github.event_name == 'pull_request' @@ -99,7 +99,7 @@ jobs: path: | ~/.m2/repository/ ~/.m2/wrapper/ - ~/.gradle/caches/ + ~/.gradle/caches/modules-2 ~/.gradle/wrapper/ key: ${{ steps.cache-key.outputs.buildtool-cache-key }} restore-keys: |