Skip to content

Commit 5584d0f

Browse files
adamfarleyadamfarley
authored
Version numbers conf parsing script (#4304)
* First draft of version numbers conf parsing script Signed-off-by: Adam Farley <[email protected]> * Second draft of the jdk version file parser This version is far more conservative so as to minimise risk of disruption to the builds. Signed-off-by: Adam Farley <[email protected]> * Linter fixes and improvements Signed-off-by: Adam Farley <[email protected]> * Checking for unbound SCM_REF Signed-off-by: Adam Farley <[email protected]> * Redelimiting unbound variable Signed-off-by: Adam Farley <[email protected]> * Making a zero-length check shorter Signed-off-by: Adam Farley <[email protected]> * Amending null check to include indexing errors Signed-off-by: Adam Farley <[email protected]> * Changing bash delimiters to disable index expansion Signed-off-by: Adam Farley <[email protected]> * Trialling removal of a flaky if condition Signed-off-by: Adam Farley <[email protected]> * Adding quotes for string Signed-off-by: Adam Farley <[email protected]> * Fix for version string parser in build.sh Signed-off-by: Adam Farley <[email protected]> * Refactoring unbound variable handling Signed-off-by: Adam Farley <[email protected]> * Removing superfluous dollar sign Signed-off-by: Adam Farley <[email protected]> * Switching to -v for cross-compatibility Signed-off-by: Adam Farley <[email protected]> * Switching to -z again Signed-off-by: Adam Farley <[email protected]> * Switching to -v Signed-off-by: Adam Farley <[email protected]> * Another try Signed-off-by: Adam Farley <[email protected]> * Trying a for loop Signed-off-by: Adam Farley <[email protected]> * Adding formatting whitespace Signed-off-by: Adam Farley <[email protected]> * Changing default build number to 1 in version string Signed-off-by: Adam Farley <[email protected]> * Handling an incorrect number of arguments Signed-off-by: Adam Farley <[email protected]> * Updating arg validation logic and comments Signed-off-by: Adam Farley <[email protected]> * Switching build_src to camel case Signed-off-by: Adam Farley <[email protected]> * Updated comments Signed-off-by: Adam Farley <[email protected]> * Removing duplicate error reporting logic Signed-off-by: Adam Farley <[email protected]> * Extra comments and restructuring for readability Signed-off-by: Adam Farley <[email protected]> * Extra comments for clarity Signed-off-by: Adam Farley <[email protected]> * Useful comment Signed-off-by: Adam Farley <[email protected]> * Extracting file parsing logic from compareToOpenJDKFileVersion Signed-off-by: Adam Farley <[email protected]> * Removing unused variable Signed-off-by: Adam Farley <[email protected]> * Switching to -n to avoid scm_ref unbound variable issue Signed-off-by: Adam Farley <[email protected]> * Switching version file parser gate to TAG for better locality Signed-off-by: Adam Farley <[email protected]> * Updating jdk version comparison logic for brevity and clarity Signed-off-by: Adam Farley <[email protected]> * Changing if to elif to prevent multiple matches Signed-off-by: Adam Farley <[email protected]> * Quoting grep output to prevent splitting Signed-off-by: Adam Farley <[email protected]> * Adding warning message for jdk version override Signed-off-by: Adam Farley <[email protected]> * Removing excess whitespace Signed-off-by: Adam Farley <[email protected]> * Updating SBOM validator to better handle false negatives Signed-off-by: Adam Farley <[email protected]> * Restructuring SHA check in SBOM validation Signed-off-by: Adam Farley <[email protected]> * Improving posix compliance of SBOM validation Signed-off-by: Adam Farley <[email protected]> * Changing script to use empty strings instead of nulls Signed-off-by: Adam Farley <[email protected]> * Amending curl command for efficiency Signed-off-by: Adam Farley <[email protected]> * Allow for empty scm parameter Signed-off-by: Adam Farley <[email protected]> * Removing unnecessary code Signed-off-by: Adam Farley <[email protected]> * Make code shorter and easier to read Signed-off-by: Adam Farley <[email protected]> * Making code more concise and easier to maintain Signed-off-by: Adam Farley <[email protected]> --------- Signed-off-by: Adam Farley <[email protected]>
1 parent a44863d commit 5584d0f

File tree

3 files changed

+124
-21
lines changed

3 files changed

+124
-21
lines changed

‎sbin/build.sh‎

Lines changed: 101 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ configureDevKitConfigureParameter() {
105105
addConfigureArg "--with-devkit=" "${BUILD_CONFIG[ADOPTIUM_DEVKIT_LOCATION]}"
106106
fi
107107
fi
108-
}
108+
}
109109

110110
# Configure the boot JDK
111111
configureBootJDKConfigureParameter() {
@@ -253,6 +253,98 @@ patchFreetypeWindows() {
253253
fi
254254
}
255255

256+
# Returns the version numbers in version-numbers.conf as a space-seperated list of integers.
257+
# e.g. 17 0 1 5
258+
# or 8 0 0 432
259+
# Build number will not be included, as that is not stored in this file.
260+
versionNumbersFileParser() {
261+
funcName="versionNumbersFileParser"
262+
buildSrc="${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/${BUILD_CONFIG[OPENJDK_SOURCE_DIR]}"
263+
jdkVersion="${BUILD_CONFIG[OPENJDK_FEATURE_NUMBER]}"
264+
265+
# Find version-numbers.conf (or equivalent) and confirm we can read it.
266+
numbersFile="${buildSrc}/common/autoconf/version-numbers"
267+
[ "$jdkVersion" -eq 11 ] && numbersFile="${buildSrc}/make/autoconf/version-numbers"
268+
[ "$jdkVersion" -ge 17 ] && numbersFile="${buildSrc}/make/conf/version-numbers.conf"
269+
[ ! -r "${numbersFile}" ] && echo "ERROR: build.sh: ${funcName}: JDK version file not found: ${numbersFile}" >&2 && exit 1
270+
271+
fileVersionString=""
272+
error=""
273+
if [ "$jdkVersion" -eq 8 ]; then
274+
# jdk8 uses this format: jdk8u482-b01
275+
fileVersionString="jdk8u$(grep "JDK_UPDATE_VERSION" "${numbersFile}" | head -1 | grep -Eo '[0-9]+')" || error="true"
276+
[[ ! "${fileVersionString}" =~ ^jdk8u[0-9]+$ || $error ]] && echo "ERROR: build.sh: ${funcName}: version file could not be parsed." >&2 && exit 1
277+
else
278+
# File parsing logic for jdk11+.
279+
patchNo="$(grep "DEFAULT_VERSION_PATCH" "${numbersFile}" | head -1 | grep -Eo '[0-9]+')" || error="true"
280+
updateNo="$(grep "DEFAULT_VERSION_UPDATE" "${numbersFile}" | head -1 | grep -Eo '[0-9]+')" || error="true"
281+
interimNo="$(grep "DEFAULT_VERSION_INTERIM" "${numbersFile}" | head -1 | grep -Eo '[0-9]+')" || error="true"
282+
featureNo="$(grep "DEFAULT_VERSION_FEATURE" "${numbersFile}" | head -1 | grep -Eo '[0-9]+')" || error="true"
283+
284+
[[ ! "${patchNo}" =~ ^0$ ]] && fileVersionString=".${patchNo}"
285+
[[ ! "${updateNo}.${fileVersionString}" =~ ^[0\.]+$ ]] && fileVersionString=".${updateNo}${fileVersionString}"
286+
[[ ! "${interimNo}.${fileVersionString}" =~ ^[0\.]+$ ]] && fileVersionString=".${interimNo}${fileVersionString}"
287+
fileVersionString="jdk-${featureNo}${fileVersionString}"
288+
289+
[[ ! "${fileVersionString}" =~ ^jdk\-[0-9]+[0-9\.]*$ || $error ]] && echo "ERROR: build.sh: ${funcName}: version file could not be parsed." >&2 && exit 1
290+
fi
291+
292+
# Returning the formatted jdk version string.
293+
echo "${fileVersionString}"
294+
}
295+
296+
# This function attempts to compare the jdk version from version-numbers.conf with arg 1.
297+
# We will then return whichever version is bigger/later.
298+
# e.g. 17.0.1+32 > 17.0.0+64
299+
# If any errors or unusual circumstances are detected, we simply return arg1 to avoid destabilising the build.
300+
# Note: For error messages, use 'echo message >&2' to ensure the error isn't intercepted by the subshell.
301+
compareToOpenJDKFileVersion() {
302+
funcName="compareToOpenJDKFileVersion"
303+
# First, sanity checking on the arg.
304+
if [ $# -eq 0 ]; then
305+
echo "compareToOpenJDKFileVersion_was_called_with_no_args"
306+
exit 1
307+
elif [ $# -gt 1 ]; then
308+
echo "ERROR: build.sh: ${funcName}: Too many arguments (>1) were passed to this function." >&2
309+
echo "$1"
310+
exit 1
311+
fi
312+
313+
# Check if arg 1 looks like a jdk version string.
314+
# Example: JDK11+: jdk-21.0.10+2
315+
# Example: JDK8 : jdk8u482-b01
316+
if [[ ! "$1" =~ ^jdk\-[0-9]+[0-9\.]*(\+[0-9]+)?$ ]]; then
317+
if [[ ! "$1" =~ ^jdk8u[0-9]+(\-b[0-9][0-9]+)?$ ]]; then
318+
echo "ERROR: build.sh: ${funcName}: The JDK version passed to this function did not match the expected format." >&2
319+
echo "$1"
320+
exit 1
321+
fi
322+
fi
323+
324+
# Retrieve the jdk version from version-numbers.conf (minus the build number).
325+
if ! fileVersionString="$(versionNumbersFileParser)"; then
326+
# This is to catch versionNumbersFileParser failures.
327+
echo "ERROR: build.sh: ${funcName}: versionNumbersFileParser has failed." >&2
328+
echo "$1"
329+
exit 1
330+
fi
331+
332+
if [[ "$1" =~ ^${fileVersionString}.*$ ]]; then
333+
# The file version matches the function argument.
334+
echo "$1"
335+
return
336+
fi
337+
338+
# The file version does not match the function argument.
339+
# Returning the file version.
340+
[ "${BUILD_CONFIG[OPENJDK_FEATURE_NUMBER]}" -eq 8 ] && fileVersionString+="-b00"
341+
[ "${BUILD_CONFIG[OPENJDK_FEATURE_NUMBER]}" -gt 8 ] && fileVersionString+="+0"
342+
343+
echo "WARNING: build.sh: ${funcName}: JDK version in source does not match the supplied version (likely the latest git tag)." >&2
344+
echo "WARNING: The JDK version in source will be used instead." >&2
345+
echo "${fileVersionString}"
346+
}
347+
256348
getOpenJdkVersion() {
257349
local version
258350

@@ -2074,7 +2166,14 @@ getOpenJDKTag() {
20742166
echo "${BUILD_CONFIG[BRANCH]}"
20752167
else
20762168
echo " getOpenJDKTag(): Determining tag from checked out repository.." 1>&2
2077-
getFirstTagFromOpenJDKGitRepo
2169+
tagString=$(getFirstTagFromOpenJDKGitRepo)
2170+
# Now we check if the version in the code is later than the version we have so far.
2171+
# This prevents an issue where the git repo tags do not match the version string in the source.
2172+
# Without this code, we can create builds where half of the version strings do not match.
2173+
# This results in nonsensical -version output, incorrect folder names, and lots of failures
2174+
# relating to those two factors.
2175+
tagString=$(compareToOpenJDKFileVersion "$tagString")
2176+
echo "${tagString}"
20782177
fi
20792178
}
20802179

‎tooling/validateSBOM.sh‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -69,11 +69,6 @@ arg_parser() {
6969
exit 1
7070
fi
7171

72-
if [ -z "$SOURCE_TAG" ]; then
73-
echo "ERROR: validateSBOM.sh: second argument must not be empty."
74-
exit 1
75-
fi
76-
7772
TAG_CHECK=""
7873
if [ "$JDK_MAJOR_VERSION" -eq "8" ]; then
7974
echo "$SOURCE_TAG" | grep -q -e "^jdk8u[0-9][0-9]*-b[0-9][0-9]*_adopt\$" \
@@ -96,7 +91,7 @@ arg_parser() {
9691
echo "WARNING: validateSBOM.sh: SOURCE_TAG does not use a valid upstream tag structure."
9792
echo "INFO: validateSBOM.sh: Build is presumed to be a personal or dev build."
9893
echo "INFO: validateSBOM.sh: SCM and SHA checks will be skipped."
99-
SOURCE_TAG="null"
94+
SOURCE_TAG=""
10095
fi
10196

10297
if [ -z "$SBOM_LOCATION" ]; then
@@ -206,7 +201,7 @@ validate_sbom_cyclonedx() {
206201
########################################################################################################################
207202
validate_sbom_content() {
208203
# shellcheck disable=SC2086
209-
echo "validateSBOM.sh: Running validateSBOMcontent.sh"
204+
echo "validateSBOM.sh: Running command: sh validateSBOMcontent.sh \"$SBOM_LOCATION\" \"$JDK_MAJOR_VERSION\" \"$SOURCE_TAG\""
210205

211206
if sh "${SCRIPT_DIR}/validateSBOMcontent.sh" "$SBOM_LOCATION" "$JDK_MAJOR_VERSION" "$SOURCE_TAG"; then
212207
echo "validateSBOMcontent.sh: PASSED"

‎tooling/validateSBOMcontent.sh‎

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -111,8 +111,8 @@ fi
111111

112112
RC=0
113113

114-
# Skip SCM check if EXPECTED_SCM_REF parameter is null
115-
if [ "${EXPECTED_SCM_REF}" != "null" ]; then
114+
# Skip SCM check if EXPECTED_SCM_REF parameter is empty
115+
if [ -n "${EXPECTED_SCM_REF}" ]; then
116116
[ "${EXPECTED_SCM_REF}" != "${SCM_REF}" ] && echo "ERROR: SCM_REF not ${EXPECTED_SCM_REF} (SBOM has ${SCM_REF})" && RC=1
117117
fi
118118
if echo "$SBOMFILE" | grep 'linux_'; then
@@ -131,18 +131,27 @@ echo "BOOTJDK is ${BOOTJDK}"
131131
echo "FREETYPE is ${FREETYPE}"
132132
# shellcheck disable=SC3037
133133
echo -n "Checking for JDK source SHA validity: "
134-
GITSHA=$(jq '.components[].properties[] | select(.name|test("OpenJDK Source Commit")) | .value' "$1" | tr -d \" | uniq)
135-
GITREPO=$(echo "$GITSHA" | cut -d/ -f1-5)
136-
GITSHA=$( echo "$GITSHA" | cut -d/ -f7)
137-
if ! git ls-remote "${GITREPO}" | grep "${GITSHA}"; then
138-
echo "ERROR: git sha of source repo not found"
139-
echo "GITREPO: ${GITREPO}"
140-
echo "GITSHA: ${GITSHA}"
141-
RC=1
134+
GITURL=$(jq '.components[].properties[] | select(.name|test("OpenJDK Source Commit")) | .value' "$1" | tr -d \" | uniq)
135+
GITREPO=$(echo "$GITURL" | cut -d/ -f1-5)
136+
GITSHA=$( echo "$GITURL" | cut -d/ -f7)
137+
if [ -z "${EXPECTED_SCM_REF}" ]; then
138+
if ! curl --silent --fail -I "$GITURL" > /dev/null; then
139+
echo "ERROR: git sha of source commit not found"
140+
echo "GITREPO: ${GITREPO}"
141+
echo "GITSHA: ${GITSHA}"
142+
RC=1
143+
fi
144+
else
145+
if ! git ls-remote "${GITREPO}" | grep "${GITSHA}"; then
146+
echo "ERROR: git sha of source repo not found"
147+
echo "GITREPO: ${GITREPO}"
148+
echo "GITSHA: ${GITSHA}"
149+
RC=1
150+
fi
142151
fi
143152

144153
# shellcheck disable=SC3037
145-
if [ "${EXPECTED_SCM_REF}" != "null" ]; then
154+
if [ -n "${EXPECTED_SCM_REF}" ]; then
146155
echo -n "Checking for temurin-build SHA validity: "
147156
GITSHA=$(jq '.components[].properties[] | select(.name|test("Temurin Build Ref")) | .value' "$1" | tr -d \" | uniq)
148157
GITREPO=$(echo "$GITSHA" | cut -d/ -f1-5)
@@ -160,7 +169,7 @@ if [ "${EXPECTED_SCM_REF}" != "null" ]; then
160169
fi
161170
fi
162171
else
163-
echo "The SCM_REF argument was set to null; skipping SHA check."
172+
echo "The SCM_REF argument was set to an empty string; skipping SHA check."
164173
fi
165174

166175
if [ "$RC" != "0" ]; then

0 commit comments

Comments
 (0)