From 99243d228d4bb2b3f5fd3e6c15122389dede0f51 Mon Sep 17 00:00:00 2001 From: Hannes Wellmann Date: Mon, 3 Mar 2025 20:46:57 +0100 Subject: [PATCH] [Build] Rename library files if native sources of platform are unchanged Rust rename the SWT native binary files for unchanged platforms to the new version and only build those whose native sources really changed. Avoiding a rebuild of effectively unchanged binaries has multiple advantages, if only a sub-set of all supported platforms is build: - reduced build times - reduced occupation of precious native agents in Jenkins - increased stability of the build because if a native build-agent for a skipped platforms is unavailable the build isn't blocked. - reduced future growth of the git large-file storage occupation as renaming a binary file has basically zero-cost, while a modified binary (even if just slightly) will add the entire file again. --- Jenkinsfile | 59 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9b10d262aa0..a7ababef5f3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,8 +42,8 @@ def runOnNativeBuildAgent(String platform, Closure body) { } } -/** Returns the download URL of the JDK against whoose C headers (in the 'include/' folder) and native libraries the SWT natives are compiled.*/ -def getNativeJdkUrl(String os, String arch){ // To update the used JDK version update the URL template below +/** Returns the download URL of the JDK against whose C headers (in the 'include/' folder) and native libraries the SWT natives are compiled.*/ +def getNativeJdkUrl(String os, String arch) { // To update the used JDK version update the URL template below if('win32'.equals(os) && 'aarch64'.equals(arch)) { // Temporary workaround until there are official Temurin GA releases for Windows on ARM that can be consumed through JustJ dir("${WORKSPACE}/repackage-win32.aarch64-jdk") { @@ -82,7 +82,7 @@ def getSWTVersions() { // must be called from the repository root return props } -boolean NATIVES_CHANGED = false +def Set NATIVES_CHANGED = [] pipeline { options { @@ -103,7 +103,9 @@ pipeline { PR_VALIDATION_BUILD = "true" } parameters { - booleanParam(name: 'forceNativeBuilds', defaultValue: false, description: 'Forces to run the native builds of swt\'s binaries. Will push the built binaries to the master branch, unless \'skipCommit\' is set. Useful in debugging.') + booleanParam(name: 'forceNativeBuilds-cocoa', defaultValue: false, description: 'Enforce a re-build of SWT\'s native binaries for Mac OS X. Will push the built binaries to the master branch, unless \'skipCommit\' is set.') + booleanParam(name: 'forceNativeBuilds-gtk', defaultValue: false, description: 'Enforce a re-build of SWT\'s native binaries for Linux. Will push the built binaries to the master branch, unless \'skipCommit\' is set.') + booleanParam(name: 'forceNativeBuilds-win32', defaultValue: false, description: 'Enforce a re-build of SWT\'s native binaries for Windows. Will push the built binaries to the master branch, unless \'skipCommit\' is set.') booleanParam(name: 'skipCommit', defaultValue: false, description: 'Stops committing to swt and swt binaries repo at the end. Useful in debugging.') } stages { @@ -141,9 +143,10 @@ pipeline { git config --global user.name 'Eclipse Releng Bot' ''' script { + def allWS = ['cocoa', 'gtk', 'win32'] + def libraryFilePattern = [ 'cocoa' : 'libswt-*.jnilib', 'gtk' : 'libswt-*.so', 'win32' : 'swt-*.dll' ] def swtTag = getLatestGitTag() echo "Current tag=${swtTag}." - boolean nativesChanged = false dir('bundles/org.eclipse.swt') { // Verify preprocessing is completed sh ''' @@ -154,28 +157,42 @@ pipeline { ''' def sourceFoldersProps = readProperties(file: 'nativeSourceFolders.properties') def sources = sourceFoldersProps.collectEntries{ k, src -> [ k, src.split(',').collect{ f -> '\'' + f + '\''}.join(' ') ] } - def diff = sh(script: "git diff HEAD ${swtTag} ${sources.values().join(' ')}", returnStdout: true) - nativesChanged = !diff.strip().isEmpty() - echo "Natives changed since ${swtTag}: ${nativesChanged}" + for(ws in allWS) { + def diff = sh(script: "git diff HEAD ${swtTag} ${sources.src_common} ${sources['src_' + ws]}", returnStdout: true) + if (!diff.strip().isEmpty()) { + NATIVES_CHANGED += ws + } + } + echo "Natives changed since ${swtTag}: ${NATIVES_CHANGED.isEmpty() ? 'None': NATIVES_CHANGED}" } - if (nativesChanged || params.forceNativeBuilds) { - NATIVES_CHANGED = true + NATIVES_CHANGED += allWS.findAll{ ws -> params[ 'forceNativeBuilds-' + ws] } + if (!NATIVES_CHANGED.isEmpty()) { def versions = getSWTVersions() sh """ - # Delete native binaries to be replaced by subsequent binaries build - rm binaries/org.eclipse.swt.gtk.*/libswt-*.so - rm binaries/org.eclipse.swt.win32.*/swt-*.dll - rm binaries/org.eclipse.swt.cocoa.*/libswt-*.jnilib - echo "Incrementing version from ${versions.swt_version} to ${versions.new_version}" sed -i -e "s/REVISION = ${versions.rev}/REVISION = ${versions.new_rev}/g" \ 'bundles/org.eclipse.swt/Eclipse SWT PI/common/org/eclipse/swt/internal/Library.java' sed -i -e "s/rev=${versions.rev}/rev=${versions.new_rev}/g" \ 'bundles/org.eclipse.swt/Eclipse SWT/common/library/make_common.mak' """ + for(ws in allWS) { + if (NATIVES_CHANGED.contains(ws)) { + sh """ + # Delete native binaries to be replaced by subsequent binaries build + rm binaries/org.eclipse.swt.${ws}.*/${libraryFilePattern[ws]} + """ + } else { + // Just rename existing native library files to new revision instead of rebuilding them + sh """ + for f in binaries/org.eclipse.swt.${ws}.*/${libraryFilePattern[ws]}; do + mv "\$f" "\$(echo \$f | sed 's/-${ws}-${versions.swt_version}/-${ws}-${versions.new_version}/g')" + done + """ + } + } // Collect SWT-native's sources dir('bundles/org.eclipse.swt') { - for (ws in ['cocoa', 'gtk', 'win32']) { + for (ws in NATIVES_CHANGED) { sh "java -Dws=${ws} build-scripts/CollectSources.java -nativeSources 'target/natives-build-temp/${ws}'" dir("target/natives-build-temp/${ws}") { stash(name:"swt.binaries.sources.${ws}") @@ -207,8 +224,14 @@ pipeline { 'win32.win32.x86_64' } } + when { + expression { + def (ws, os, arch) = env.PLATFORM.split('\\.') + return NATIVES_CHANGED.any{ w -> ws.startsWith(w)} // handle also dedicated gtk4 build + } + } stages { - stage('Build SWT-natives') { + stage('Initiate native build') { options { timeout(time: 120, unit: 'MINUTES') // Some build agents are rare and it might take awhile until they are available. } @@ -229,7 +252,7 @@ pipeline { unstash "jdk.resources.${os}.${arch}" } withEnv(['MODEL=' + arch, "OUTPUT_DIR=${WORKSPACE}/libs", "SWT_JAVA_HOME=${WORKSPACE}/jdk.resources"]) { - if (isUnix()){ + if (isUnix()) { sh '''#!/bin/bash -x mkdir libs if [[ ${PLATFORM} == gtk.linux.* ]]; then