Skip to content

Commit 96e14e1

Browse files
sophia-guosophia-guo
authored
Enable nightly build with reproducible comparison (#714)
* Add detail error messages Signed-off-by: Sophia Guo <[email protected]> * Enable nightly reproducible comparison Resubmit the changes Signed-off-by: Sophia Guo <[email protected]> * Correct lint error Signed-off-by: Sophia Guo <[email protected]> * Add enableReproducibleCompare flag Signed-off-by: Sophia Guo <[email protected]> * Set reproducible job ENABLE_REPRODUCIBLE_COMPARE false Avoid recursive build Signed-off-by: Sophia Guo <[email protected]> * Update build timestamp Signed-off-by: Sophia Guo <[email protected]> * Format update Signed-off-by: Sophia Guo <[email protected]> * Update test case Signed-off-by: Sophia Guo <[email protected]> * Remove unnecessary file permission step Signed-off-by: Sophia Guo <[email protected]> * Typo Signed-off-by: Sophia Guo <[email protected]> * Fix isRelease and params correctly Signed-off-by: Sophia Guo <[email protected]> * Limit json file to SBOM specific and remove unnecessary code Signed-off-by: Sophia Guo <[email protected]> --------- Signed-off-by: Sophia Guo <[email protected]>
1 parent 2f135ca commit 96e14e1

17 files changed

+218
-64
lines changed

‎docs/UsingOurScripts.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,8 @@ This file contains the default constants and paths used in the build scripts for
7878
},
7979
// Test suite and inclusion details
8080
"testDetails" : {
81+
// Boolean denoting whether the reproducible compare is needed
82+
"enableReproducibleCompare" : false,
8183
// Boolean denoting whether pipeline tests will be enabled or disabled by default
8284
"enableTests" : true,
8385
/*

‎pipelines/build/common/build_base_file.groovy‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ class Builder implements Serializable {
3939
Map<String, ?> DEFAULTS_JSON
4040
String activeNodeTimeout
4141
Map<String, List<String>> dockerExcludes
42+
boolean enableReproducibleCompare
4243
boolean enableTests
4344
boolean enableTestDynamicParallel
4445
boolean enableInstallers
@@ -122,7 +123,7 @@ class Builder implements Serializable {
122123
if (additionalBuildArgs) {
123124
buildArgs += ' ' + additionalBuildArgs
124125
}
125-
126+
def enableReproducibleCompare = getReproducibleCompare(platformConfig, variant)
126127
def testList = getTestList(platformConfig, variant)
127128

128129
def dynamicTestsParameters = getDynamicParams(platformConfig, variant)
@@ -178,6 +179,7 @@ class Builder implements Serializable {
178179
RELEASE: release,
179180
PUBLISH_NAME: publishName,
180181
ADOPT_BUILD_NUMBER: adoptBuildNumber,
182+
ENABLE_REPRODUCIBLE_COMPARE: enableReproducibleCompare,
181183
ENABLE_TESTS: enableTests,
182184
ENABLE_TESTDYNAMICPARALLEL: enableTestDynamicParallel,
183185
ENABLE_INSTALLERS: enableInstallers,
@@ -213,6 +215,22 @@ class Builder implements Serializable {
213215

214216
return ''
215217
}
218+
/*
219+
Get reproduciableCompare flag from the build configurations.
220+
*/
221+
Boolean getReproducibleCompare(Map<String, ?> configuration, String variant) {
222+
Boolean enableReproducibleCompare = DEFAULTS_JSON['testDetails']['enableReproducibleCompare'] as Boolean
223+
if (configuration.containsKey('reproducibleCompare')) {
224+
def reproducibleCompare
225+
if (isMap(configuration.reproducibleCompare)) {
226+
reproducibleCompare = (configuration.enableReproducibleCompare as Map).get(variant)
227+
}
228+
if (reproducibleCompare != null) {
229+
enableReproducibleCompare = reproducibleCompare
230+
}
231+
}
232+
return enableReproducibleCompare
233+
}
216234

217235
/*
218236
Get the list of tests to run from the build configurations.
@@ -781,6 +799,7 @@ class Builder implements Serializable {
781799

782800
context.echo "Java: ${javaToBuild}"
783801
context.echo "OS: ${targetConfigurations}"
802+
context.echo "Enable reproducible compare: ${enableReproducibleCompare}"
784803
context.echo "Enable tests: ${enableTests}"
785804
context.echo "Enable Installers: ${enableInstallers}"
786805
context.echo "Enable Signer: ${enableSigner}"
@@ -925,6 +944,7 @@ return {
925944
Map<String, ?> DEFAULTS_JSON,
926945
String activeNodeTimeout,
927946
String dockerExcludes,
947+
String enableReproducibleCompare,
928948
String enableTests,
929949
String enableTestDynamicParallel,
930950
String enableInstallers,
@@ -989,6 +1009,7 @@ return {
9891009
DEFAULTS_JSON: DEFAULTS_JSON,
9901010
activeNodeTimeout: activeNodeTimeout,
9911011
dockerExcludes: buildsExcludeDocker,
1012+
enableReproducibleCompare: Boolean.parseBoolean(enableReproducibleCompare),
9921013
enableTests: Boolean.parseBoolean(enableTests),
9931014
enableTestDynamicParallel: Boolean.parseBoolean(enableTestDynamicParallel),
9941015
enableInstallers: Boolean.parseBoolean(enableInstallers),

‎pipelines/build/common/config_regeneration.groovy‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,24 @@ class Regeneration implements Serializable {
334334

335335
return ''
336336
}
337+
/*
338+
* Get reproduciableCompare flag from jdk*_pipeline_config.groovy. Used when creating the IndividualBuildConfig.
339+
* @param configuration
340+
* @param variant
341+
*/
342+
Boolean getReproducibleCompare(Map<String, ?> configuration, String variant) {
343+
Boolean enableReproducibleCompare = DEFAULTS_JSON['testDetails']['enableReproducibleCompare'] as Boolean
344+
if (configuration.containsKey('reproducibleCompare')) {
345+
def reproducibleCompare
346+
if (isMap(configuration.reproducibleCompare)) {
347+
reproducibleCompare = (configuration.enableReproducibleCompare as Map).get(variant)
348+
}
349+
if (reproducibleCompare != null) {
350+
enableReproducibleCompare = reproducibleCompare
351+
}
352+
}
353+
return enableReproducibleCompare
354+
}
337355

338356
/*
339357
* Get the list of tests from jdk*_pipeline_config.groovy. Used when creating the IndividualBuildConfig. Used as a placeholder since the pipelines overwrite this.
@@ -425,6 +443,8 @@ class Regeneration implements Serializable {
425443

426444
def numMachines = getDynamicParams().get('numMachines')
427445

446+
def enableReproducibleCompare = getReproducibleCompare(platformConfig, variant)
447+
428448
return new IndividualBuildConfig( // final build config
429449
JAVA_TO_BUILD: javaToBuild,
430450
ARCHITECTURE: platformConfig.arch as String,
@@ -460,6 +480,7 @@ class Regeneration implements Serializable {
460480
RELEASE: false,
461481
PUBLISH_NAME: '',
462482
ADOPT_BUILD_NUMBER: '',
483+
ENABLE_REPRODUCIBLE_COMPARE: enableReproducibleCompare,
463484
ENABLE_TESTS: DEFAULTS_JSON['testDetails']['enableTests'] as Boolean,
464485
ENABLE_TESTDYNAMICPARALLEL: DEFAULTS_JSON['testDetails']['enableTestDynamicParallel'] as Boolean,
465486
ENABLE_INSTALLERS: true,

‎pipelines/build/common/create_job_from_template.groovy‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@ pipelineJob("$buildFolder/$JOB_NAME") {
128128
<dt><strong>RELEASE</strong></dt><dd>Is this build a release</dd>
129129
<dt><strong>PUBLISH_NAME</strong></dt><dd>Set name of publish</dd>
130130
<dt><strong>ADOPT_BUILD_NUMBER</strong></dt><dd>Adopt build number</dd>
131+
<dt><strong>ENABLE_REPRODUCIBLE_COMPARE</strong></dt><dd>Run reproducible compare build</dd>
131132
<dt><strong>ENABLE_TESTS</strong></dt><dd>Run tests</dd>
132133
<dt><strong>ENABLE_TESTDYNAMICPARALLEL</strong></dt><dd>Run parallel</dd>
133134
<dt><strong>ENABLE_INSTALLERS</strong></dt><dd>Run installers</dd>

‎pipelines/build/common/openjdk_build_pipeline.groovy‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -585,6 +585,33 @@ class Build {
585585
return remoteTargets
586586
}
587587

588+
def compareReproducibleBuild() {
589+
// Currently only enable for jdk17, linux_x64, temurin, nightly, which shouldn't affect current build
590+
// Move out of normal jdk** folder as it won't be regenerated automatically right now
591+
def jobName = "${env.JOB_NAME}"
592+
jobName = jobName.substring(jobName.lastIndexOf('/')+1)
593+
jobName = "${jobName}_reproduce_compare"
594+
if (getJavaVersionNumber() == 17 &&
595+
buildConfig.ARCHITECTURE.contains('x64') &&
596+
buildConfig.TARGET_OS.contains('linux') &&
597+
buildConfig.VARIANT == 'temurin' &&
598+
!Boolean.valueOf(buildConfig.RELEASE)) {
599+
// For now set the build as independent, no need to wait for result as the build takes time
600+
context.stage('Reproduce Compare') {
601+
def buildParams = context.params.toString()
602+
// passing buildParams multiline parameter to downstream job, double check the available method
603+
context.build job: jobName,
604+
propagate: false,
605+
parameters: [
606+
context.string(name: 'COMPARED_JOB_NUMBER', value: "${env.BUILD_NUMBER}"),
607+
context.string(name: 'COMPARED_JOB_NAME', value: "${env.JOB_NAME}"),
608+
context.string(name: 'COMPARED_JOB_PARAMS', value: buildParams)
609+
],
610+
wait: false
611+
}
612+
}
613+
}
614+
588615
/*
589616
We use this function at the end of a build to parse a java version string and create a VersionInfo object for deployment in the metadata objects.
590617
E.g. 11.0.9+10-202010192351 would be one example of a matched string.
@@ -1699,7 +1726,7 @@ class Build {
16991726
context.println "Executing tests: ${buildConfig.TEST_LIST}"
17001727
context.println "Build num: ${env.BUILD_NUMBER}"
17011728
context.println "File name: ${filename}"
1702-
1729+
def enableReproducibleCompare = Boolean.valueOf(buildConfig.ENABLE_REPRODUCIBLE_COMPARE)
17031730
def enableTests = Boolean.valueOf(buildConfig.ENABLE_TESTS)
17041731
def enableInstallers = Boolean.valueOf(buildConfig.ENABLE_INSTALLERS)
17051732
def enableSigner = Boolean.valueOf(buildConfig.ENABLE_SIGNER)
@@ -1886,6 +1913,10 @@ class Build {
18861913
}
18871914
}
18881915

1916+
// Compare reproducible build if needed
1917+
if (enableReproducibleCompare) {
1918+
compareReproducibleBuild()
1919+
}
18891920
// Run Smoke Tests and AQA Tests
18901921
if (enableTests) {
18911922
try {
@@ -1913,7 +1944,7 @@ class Build {
19131944
context.parallel testStages
19141945
}
19151946
} else {
1916-
context.println("[ERROR]Smoke tests are not successful! AQA and Tck tests are blocked ")
1947+
context.println('[ERROR]Smoke tests are not successful! AQA and Tck tests are blocked ')
19171948
}
19181949
} catch (Exception e) {
19191950
context.println(e.message)

‎pipelines/build/openjdk_pipeline.groovy‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,7 @@ if (scmVars != null || configureBuild != null || buildConfigurations != null) {
150150
DEFAULTS_JSON,
151151
activeNodeTimeout,
152152
dockerExcludes,
153+
enableReproducibleCompare,
153154
enableTests,
154155
enableTestDynamicParallel,
155156
enableInstallers,

‎pipelines/build/prTester/pr_test_pipeline.groovy‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ class PullRequestTestPipeline implements Serializable {
5252
CHECKOUT_CREDENTIALS: '',
5353
adoptScripts : true,
5454
enableTests : false,
55+
enableReproducibleCompare : false,
5556
enableTestDynamicParallel : false,
5657
releaseType : "pr-tester"
5758
]

‎pipelines/build/regeneration/build_job_generator.groovy‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,19 +49,19 @@ node('worker') {
4949

5050
try {
5151
// Load git url and branch and gitBranch. These determine where we will be pulling configs from.
52-
def repoUri = (params.REPOSITORY_URL) ?: DEFAULTS_JSON['repository']['pipeline_url']
52+
def repoUrl = (params.REPOSITORY_URL) ?: DEFAULTS_JSON['repository']['pipeline_url']
5353
def repoBranch = (params.REPOSITORY_BRANCH) ?: DEFAULTS_JSON['repository']['pipeline_branch']
5454

5555
// Load credentials to be used in checking out. This is in case we are checking out a URL that is not Adopts and they don't have their ssh key on the machine.
5656
def checkoutCreds = (params.CHECKOUT_CREDENTIALS) ?: ''
5757
def remoteConfigs = new JsonSlurper().parseText('{ "url": "" }') as Map
58-
remoteConfigs.url = repoUri
58+
remoteConfigs.url = repoUrl
5959

6060
if (checkoutCreds != '') {
6161
// This currently does not work with user credentials due to https://issues.jenkins.io/browse/JENKINS-60349
6262
remoteConfigs.credentials = "${checkoutCreds}"
6363
} else {
64-
println "[WARNING] CHECKOUT_CREDENTIALS not specified! Checkout to $repoUri may fail if you do not have your ssh key on this machine."
64+
println "[WARNING] CHECKOUT_CREDENTIALS not specified! Checkout to $repoUrl may fail if you do not have your ssh key on this machine."
6565
}
6666

6767
/*
@@ -186,7 +186,7 @@ node('worker') {
186186

187187
println '[INFO] Running regeneration script with the following configuration:'
188188
println "VERSION: $javaVersion"
189-
println "CI REPOSITORY URL: $repoUri"
189+
println "CI REPOSITORY URL: $repoUrl"
190190
println "CI REPOSITORY BRANCH: $repoBranch"
191191
println "BUILD CONFIGURATIONS: ${JsonOutput.prettyPrint(JsonOutput.toJson(buildConfigurations))}"
192192
println "JOBS TO GENERATE: ${JsonOutput.prettyPrint(JsonOutput.toJson(targetConfigurations))}"

‎pipelines/build/regeneration/build_pipeline_generator.groovy‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ node('worker') {
209209
if (useAdoptShellScripts.toBoolean()) {
210210
config.put('adoptScripts', true)
211211
}
212-
212+
config.put('enableReproducibleCompare', DEFAULTS_JSON['testDetails']['enableReproducibleCompare'] as Boolean)
213213
config.put('enableTests', DEFAULTS_JSON['testDetails']['enableTests'] as Boolean)
214214
config.put('enableTestDynamicParallel', DEFAULTS_JSON['testDetails']['enableTestDynamicParallel'] as Boolean)
215215

‎pipelines/build/regeneration/evaluation_pipeline_generator.groovy‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -215,7 +215,7 @@ node('worker') {
215215
if (useAdoptShellScripts.toBoolean()) {
216216
config.put('adoptScripts', true)
217217
}
218-
218+
config.put('enableReproducibleCompare', DEFAULTS_JSON['testDetails']['enableReproducibleCompare'] as Boolean)
219219
config.put('enableTests', DEFAULTS_JSON['testDetails']['enableTests'] as Boolean)
220220
config.put('enableTestDynamicParallel', DEFAULTS_JSON['testDetails']['enableTestDynamicParallel'] as Boolean)
221221
config.put('defaultsJson', DEFAULTS_JSON)

0 commit comments

Comments
 (0)