From a3ea5da711142ff0dc50ba30dc354d079336162f Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Thu, 20 Feb 2025 17:04:57 -0500 Subject: [PATCH 01/34] ci: Use Linkage Checker to test Binary Compatibility --- ..._protobuf_compatibility_check_nightly.yaml | 5 +- ...ownstream-protobuf-binary-compatibility.sh | 83 +++++++++++++++++++ ...ownstream-protobuf-source-compatibility.sh | 3 +- 3 files changed, 88 insertions(+), 3 deletions(-) create mode 100755 .kokoro/nightly/downstream-protobuf-binary-compatibility.sh diff --git a/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml b/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml index a186a8527b..93b5670237 100644 --- a/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml +++ b/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml @@ -46,9 +46,12 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 with: - java-version: 17 + java-version: 11 distribution: temurin - name: Print Protobuf-Java testing version run: echo "Testing with Protobuf-Java v${{ matrix.protobuf-version }}" - name: Perform downstream source compatibility testing run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version}}" ./.kokoro/nightly/downstream-protobuf-source-compatibility.sh + - name: Perform downstream binary compatibility testing + run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version}}" ./.kokoro/nightly/downstream-protobuf-binary-compatibility.sh + diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh new file mode 100755 index 0000000000..409db40740 --- /dev/null +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +# Comma-delimited list of repos to test with the local java-shared-dependencies +if [ -z "${REPOS_UNDER_TEST}" ]; then + echo "REPOS_UNDER_TEST must be set to run downstream-protobuf-source-compatibility.sh" + echo "Expects a comma-delimited list: i.e REPOS_UNDER_TEST=\"java-bigtable,java-bigquery\"" + exit 1 +fi + +# Version of Protobuf-Java runtime to compile with +if [ -z "${PROTOBUF_RUNTIME_VERSION}" ]; then + echo "PROTOBUF_RUNTIME_VERSION must be set to run downstream-protobuf-source-compatibility.sh" + echo "Expects a single Protobuf-Java runtime version i.e. PROTOBUF_RUNTIME_VERSION=\"4.28.3\"" + exit 1 +fi + +# cloud-opensource-java contains the Linkage Checker tool +git clone https://github.com/GoogleCloudPlatform/cloud-opensource-java.git +pushd cloud-opensource-java +mvn -B -ntp clean compile +# Linkage Checker tool resides in the /dependencies subfolder +pushd dependencies + +for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma + # Perform source-compatibility testing on main (latest changes) + git clone "https://github.com/googleapis/$repo.git" --depth=1 + pushd "$repo" + # Install all the repo's modules to local ~/.m2 + mvn -B -ntp clean install -T 1C -DskipTests -Dclirr.skip + + # Match all artifacts that start with google-cloud (rules out proto and grpc modules) + # Exclude any matches to BOM artifacts or emulators + # The artifact list will look something like "com.google.cloud:google-cloud-accessapproval:2.60.0-SNAPSHOT,com.google.cloud:google-cloud-aiplatform:3.60.0-SNAPSHOT" + CLOUD_ARTIFACT_LIST=$(cat "versions.txt" | grep "^google-cloud" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="com.google.cloud:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') + CLOUD_ARTIFACT_LIST=${CLOUD_ARTIFACT_LIST%,} + if [ -n "${CLOUD_ARTIFACT_LIST}" ]; then + ARTIFACT_LIST="${ARTIFACT_LIST},${CLOUD_ARTIFACT_LIST}" + fi + + # This logic is only for google-cloud-java as there are non-cloud APIs included + # This also excludes any proto, grpc, bom, and emulators included + GRAFEAS_ARTIFACT_LIST=$(cat "versions.txt" | grep "^grafeas" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="io.grafeas:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') + GRAFEAS_ARTIFACT_LIST=${GRAFEAS_ARTIFACT_LIST%,} + MAPS_ARTIFACT_LIST=$(cat "versions.txt" | grep "^google-maps" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="com.google.maps:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') + MAPS_ARTIFACT_LIST=${MAPS_ARTIFACT_LIST%,} + SHOPPING_ARTIFACT_LIST=$(cat "versions.txt" | grep "^google-shopping" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="com.google.shopping:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') + SHOPPING_ARTIFACT_LIST=${SHOPPING_ARTIFACT_LIST%,} + ARTIFACT_LIST="" + if [ -n "${GRAFEAS_ARTIFACT_LIST}" ]; then + ARTIFACT_LIST="${ARTIFACT_LIST},${GRAFEAS_ARTIFACT_LIST}" + fi + if [ -n "${MAPS_ARTIFACT_LIST}" ]; then + ARTIFACT_LIST="${ARTIFACT_LIST},${MAPS_ARTIFACT_LIST}" + fi + if [ -n "${SHOPPING_ARTIFACT_LIST}" ]; then + ARTIFACT_LIST="${ARTIFACT_LIST},${SHOPPING_ARTIFACT_LIST}" + fi + + echo "Found artifacts ${ARTIFACT_LIST}" + popd + + # The `-s` argument filters the linkage check problems that stem from the artifact + program_args="-r --artifacts ${ARTIFACT_LIST},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${ARTIFACT_LIST}" + echo "Linkage Checker Program Arguments: ${program_args}" + mvn -B -ntp exec:java -Dexec.mainClass="com.google.cloud.tools.opensource.classpath.LinkageCheckerMain" -Dexec.args="${program_args}" +done +popd +popd diff --git a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh index c7caeb724c..41af54f81e 100755 --- a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh @@ -35,8 +35,7 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma pushd "$repo" # Compile the Handwritten Library with the Protobuf-Java version to test source compatibility - # Run unit tests to help check for any behavior differences (dependant on coverage) - mvn clean test -B -V -ntp \ + mvn clean compile -B -V -ntp \ -Dclirr.skip=true \ -Denforcer.skip=true \ -Dmaven.javadoc.skip=true \ From b55c8add3641664ef37e378720ccab2dcdadc99e Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Fri, 21 Feb 2025 11:59:05 -0500 Subject: [PATCH 02/34] ci: Fix generating non-cloud api list --- .../downstream-protobuf-binary-compatibility.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index 409db40740..2de948fa3c 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2024 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -40,13 +40,15 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma # Perform source-compatibility testing on main (latest changes) git clone "https://github.com/googleapis/$repo.git" --depth=1 pushd "$repo" - # Install all the repo's modules to local ~/.m2 + # Install all modules to ~/.m2 (there can be multiple relevant versions i.e. core, admin, control) mvn -B -ntp clean install -T 1C -DskipTests -Dclirr.skip - # Match all artifacts that start with google-cloud (rules out proto and grpc modules) - # Exclude any matches to BOM artifacts or emulators - # The artifact list will look something like "com.google.cloud:google-cloud-accessapproval:2.60.0-SNAPSHOT,com.google.cloud:google-cloud-aiplatform:3.60.0-SNAPSHOT" + ARTIFACT_LIST="" + # Match all artifacts that start with google-cloud to exclude any proto and grpc modules. + # Additionally, exclude any matches to BOM artifacts or emulators + # The artifact list will look something like "com.google.cloud:google-cloud-accessapproval:2.60.0-SNAPSHOT,com.google.cloud:google-cloud-aiplatform:3.60.0-SNAPSHOT," CLOUD_ARTIFACT_LIST=$(cat "versions.txt" | grep "^google-cloud" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="com.google.cloud:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') + # Remove the trailing comma after the last entry CLOUD_ARTIFACT_LIST=${CLOUD_ARTIFACT_LIST%,} if [ -n "${CLOUD_ARTIFACT_LIST}" ]; then ARTIFACT_LIST="${ARTIFACT_LIST},${CLOUD_ARTIFACT_LIST}" @@ -60,7 +62,6 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma MAPS_ARTIFACT_LIST=${MAPS_ARTIFACT_LIST%,} SHOPPING_ARTIFACT_LIST=$(cat "versions.txt" | grep "^google-shopping" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="com.google.shopping:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') SHOPPING_ARTIFACT_LIST=${SHOPPING_ARTIFACT_LIST%,} - ARTIFACT_LIST="" if [ -n "${GRAFEAS_ARTIFACT_LIST}" ]; then ARTIFACT_LIST="${ARTIFACT_LIST},${GRAFEAS_ARTIFACT_LIST}" fi From 20a654ce4f010dfbf074a79455ffe1be390638f2 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Fri, 21 Feb 2025 12:06:08 -0500 Subject: [PATCH 03/34] ci: Fix exit code 1 for non-matching elements --- ...ownstream-protobuf-binary-compatibility.sh | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index 2de948fa3c..7aa1be0e56 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -48,37 +48,40 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma # Additionally, exclude any matches to BOM artifacts or emulators # The artifact list will look something like "com.google.cloud:google-cloud-accessapproval:2.60.0-SNAPSHOT,com.google.cloud:google-cloud-aiplatform:3.60.0-SNAPSHOT," CLOUD_ARTIFACT_LIST=$(cat "versions.txt" | grep "^google-cloud" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="com.google.cloud:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') - # Remove the trailing comma after the last entry - CLOUD_ARTIFACT_LIST=${CLOUD_ARTIFACT_LIST%,} if [ -n "${CLOUD_ARTIFACT_LIST}" ]; then + # Remove the trailing comma after the last entry + CLOUD_ARTIFACT_LIST=${CLOUD_ARTIFACT_LIST%,} ARTIFACT_LIST="${ARTIFACT_LIST},${CLOUD_ARTIFACT_LIST}" fi # This logic is only for google-cloud-java as there are non-cloud APIs included # This also excludes any proto, grpc, bom, and emulators included GRAFEAS_ARTIFACT_LIST=$(cat "versions.txt" | grep "^grafeas" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="io.grafeas:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') - GRAFEAS_ARTIFACT_LIST=${GRAFEAS_ARTIFACT_LIST%,} - MAPS_ARTIFACT_LIST=$(cat "versions.txt" | grep "^google-maps" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="com.google.maps:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') - MAPS_ARTIFACT_LIST=${MAPS_ARTIFACT_LIST%,} - SHOPPING_ARTIFACT_LIST=$(cat "versions.txt" | grep "^google-shopping" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="com.google.shopping:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') - SHOPPING_ARTIFACT_LIST=${SHOPPING_ARTIFACT_LIST%,} if [ -n "${GRAFEAS_ARTIFACT_LIST}" ]; then + GRAFEAS_ARTIFACT_LIST=${GRAFEAS_ARTIFACT_LIST%,} ARTIFACT_LIST="${ARTIFACT_LIST},${GRAFEAS_ARTIFACT_LIST}" fi + MAPS_ARTIFACT_LIST=$(cat "versions.txt" | grep "^google-maps" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="com.google.maps:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') if [ -n "${MAPS_ARTIFACT_LIST}" ]; then + MAPS_ARTIFACT_LIST=${MAPS_ARTIFACT_LIST%,} ARTIFACT_LIST="${ARTIFACT_LIST},${MAPS_ARTIFACT_LIST}" fi + SHOPPING_ARTIFACT_LIST=$(cat "versions.txt" | grep "^google-shopping" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="com.google.shopping:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') if [ -n "${SHOPPING_ARTIFACT_LIST}" ]; then + SHOPPING_ARTIFACT_LIST=${SHOPPING_ARTIFACT_LIST%,} ARTIFACT_LIST="${ARTIFACT_LIST},${SHOPPING_ARTIFACT_LIST}" fi - echo "Found artifacts ${ARTIFACT_LIST}" + # Linkage Checker /dependencies popd - # The `-s` argument filters the linkage check problems that stem from the artifact - program_args="-r --artifacts ${ARTIFACT_LIST},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${ARTIFACT_LIST}" - echo "Linkage Checker Program Arguments: ${program_args}" - mvn -B -ntp exec:java -Dexec.mainClass="com.google.cloud.tools.opensource.classpath.LinkageCheckerMain" -Dexec.args="${program_args}" + if [ -n "${ARTIFACT_LIST}" ]; then + echo "Found artifacts ${ARTIFACT_LIST}" + # The `-s` argument filters the linkage check problems that stem from the artifact + program_args="-r --artifacts ${ARTIFACT_LIST},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${ARTIFACT_LIST}" + echo "Linkage Checker Program Arguments: ${program_args}" + mvn -B -ntp exec:java -Dexec.mainClass="com.google.cloud.tools.opensource.classpath.LinkageCheckerMain" -Dexec.args="${program_args}" + fi done popd popd From 35b4324b478156014ade26b1dcb5e9275ef0bec6 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Fri, 21 Feb 2025 12:27:32 -0500 Subject: [PATCH 04/34] ci: Fix exit code 1 for non-matching elements --- ...ownstream-protobuf-binary-compatibility.sh | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index 7aa1be0e56..edf351a6f4 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -41,35 +41,36 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma git clone "https://github.com/googleapis/$repo.git" --depth=1 pushd "$repo" # Install all modules to ~/.m2 (there can be multiple relevant versions i.e. core, admin, control) - mvn -B -ntp clean install -T 1C -DskipTests -Dclirr.skip + mvn -B -ntp clean install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip ARTIFACT_LIST="" # Match all artifacts that start with google-cloud to exclude any proto and grpc modules. # Additionally, exclude any matches to BOM artifacts or emulators # The artifact list will look something like "com.google.cloud:google-cloud-accessapproval:2.60.0-SNAPSHOT,com.google.cloud:google-cloud-aiplatform:3.60.0-SNAPSHOT," CLOUD_ARTIFACT_LIST=$(cat "versions.txt" | grep "^google-cloud" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="com.google.cloud:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') + CLOUD_ARTIFACT_LIST=${CLOUD_ARTIFACT_LIST%,} if [ -n "${CLOUD_ARTIFACT_LIST}" ]; then # Remove the trailing comma after the last entry - CLOUD_ARTIFACT_LIST=${CLOUD_ARTIFACT_LIST%,} ARTIFACT_LIST="${ARTIFACT_LIST},${CLOUD_ARTIFACT_LIST}" fi # This logic is only for google-cloud-java as there are non-cloud APIs included - # This also excludes any proto, grpc, bom, and emulators included - GRAFEAS_ARTIFACT_LIST=$(cat "versions.txt" | grep "^grafeas" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="io.grafeas:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') - if [ -n "${GRAFEAS_ARTIFACT_LIST}" ]; then + if [ "${repo}" == "google-cloud-java" ]; then + GRAFEAS_ARTIFACT_LIST=$(cat "versions.txt" | grep "^grafeas" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="io.grafeas:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') GRAFEAS_ARTIFACT_LIST=${GRAFEAS_ARTIFACT_LIST%,} - ARTIFACT_LIST="${ARTIFACT_LIST},${GRAFEAS_ARTIFACT_LIST}" - fi - MAPS_ARTIFACT_LIST=$(cat "versions.txt" | grep "^google-maps" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="com.google.maps:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') - if [ -n "${MAPS_ARTIFACT_LIST}" ]; then + if [ -n "${GRAFEAS_ARTIFACT_LIST}" ]; then + ARTIFACT_LIST="${ARTIFACT_LIST},${GRAFEAS_ARTIFACT_LIST}" + fi + MAPS_ARTIFACT_LIST=$(cat "versions.txt" | grep "^google-maps" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="com.google.maps:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') MAPS_ARTIFACT_LIST=${MAPS_ARTIFACT_LIST%,} - ARTIFACT_LIST="${ARTIFACT_LIST},${MAPS_ARTIFACT_LIST}" - fi - SHOPPING_ARTIFACT_LIST=$(cat "versions.txt" | grep "^google-shopping" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="com.google.shopping:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') - if [ -n "${SHOPPING_ARTIFACT_LIST}" ]; then + if [ -n "${MAPS_ARTIFACT_LIST}" ]; then + ARTIFACT_LIST="${ARTIFACT_LIST},${MAPS_ARTIFACT_LIST}" + fi + SHOPPING_ARTIFACT_LIST=$(cat "versions.txt" | grep "^google-shopping" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="com.google.shopping:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') SHOPPING_ARTIFACT_LIST=${SHOPPING_ARTIFACT_LIST%,} - ARTIFACT_LIST="${ARTIFACT_LIST},${SHOPPING_ARTIFACT_LIST}" + if [ -n "${SHOPPING_ARTIFACT_LIST}" ]; then + ARTIFACT_LIST="${ARTIFACT_LIST},${SHOPPING_ARTIFACT_LIST}" + fi fi # Linkage Checker /dependencies From de0b8891ddd5939c83050cf2339577c459f002ba Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 24 Feb 2025 13:58:24 -0500 Subject: [PATCH 05/34] ci: Create a map of non-cloud APIs --- ...ownstream-protobuf-binary-compatibility.sh | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index edf351a6f4..256d6e4ea7 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -29,6 +29,11 @@ if [ -z "${PROTOBUF_RUNTIME_VERSION}" ]; then exit 1 fi +declare -A nonCloudAPIs +nonCloudAPIs["grafeas"]="com.google.cloud" +nonCloudAPIs["google-maps"]="com.google.maps" +nonCloudAPIs["google-shopping"]="com.google.shopping" + # cloud-opensource-java contains the Linkage Checker tool git clone https://github.com/GoogleCloudPlatform/cloud-opensource-java.git pushd cloud-opensource-java @@ -56,21 +61,13 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma # This logic is only for google-cloud-java as there are non-cloud APIs included if [ "${repo}" == "google-cloud-java" ]; then - GRAFEAS_ARTIFACT_LIST=$(cat "versions.txt" | grep "^grafeas" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="io.grafeas:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') - GRAFEAS_ARTIFACT_LIST=${GRAFEAS_ARTIFACT_LIST%,} - if [ -n "${GRAFEAS_ARTIFACT_LIST}" ]; then - ARTIFACT_LIST="${ARTIFACT_LIST},${GRAFEAS_ARTIFACT_LIST}" - fi - MAPS_ARTIFACT_LIST=$(cat "versions.txt" | grep "^google-maps" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="com.google.maps:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') - MAPS_ARTIFACT_LIST=${MAPS_ARTIFACT_LIST%,} - if [ -n "${MAPS_ARTIFACT_LIST}" ]; then - ARTIFACT_LIST="${ARTIFACT_LIST},${MAPS_ARTIFACT_LIST}" - fi - SHOPPING_ARTIFACT_LIST=$(cat "versions.txt" | grep "^google-shopping" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="com.google.shopping:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') - SHOPPING_ARTIFACT_LIST=${SHOPPING_ARTIFACT_LIST%,} - if [ -n "${SHOPPING_ARTIFACT_LIST}" ]; then - ARTIFACT_LIST="${ARTIFACT_LIST},${SHOPPING_ARTIFACT_LIST}" - fi + for key in "${!nonCloudAPIs[@]}"; do + value="${nonCloudAPIs[$key]}" + echo "Key: $key, Value: $value" + NON_CLOUD_ARTIFACT_LIST=$(cat "versions.txt" | grep "^${key}" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="${value}:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') + NON_CLOUD_ARTIFACT_LIST=${NON_CLOUD_ARTIFACT_LIST%,} + ARTIFACT_LIST="${ARTIFACT_LIST},${NON_CLOUD_ARTIFACT_LIST}" + done fi # Linkage Checker /dependencies From fe89f01b6dd6a725a8958cdce36ee0984a0d8fd6 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 24 Feb 2025 14:27:26 -0500 Subject: [PATCH 06/34] ci: Fix string interpolation issue with single quotes --- .kokoro/nightly/downstream-protobuf-binary-compatibility.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index 256d6e4ea7..00c63ef575 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -64,7 +64,7 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma for key in "${!nonCloudAPIs[@]}"; do value="${nonCloudAPIs[$key]}" echo "Key: $key, Value: $value" - NON_CLOUD_ARTIFACT_LIST=$(cat "versions.txt" | grep "^${key}" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="${value}:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') + NON_CLOUD_ARTIFACT_LIST=$(cat "versions.txt" | grep "^${key}" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: "{\$1=\"${value}:\$1; \$2=\"\"; print}" OFS=: | sed 's/::/:/' | tr '\n' ',') NON_CLOUD_ARTIFACT_LIST=${NON_CLOUD_ARTIFACT_LIST%,} ARTIFACT_LIST="${ARTIFACT_LIST},${NON_CLOUD_ARTIFACT_LIST}" done From cd0a03b41718972ac6ae6a1bda669d7e6cbf97e8 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 24 Feb 2025 14:51:36 -0500 Subject: [PATCH 07/34] ci: Fix awk command --- .../downstream-protobuf-binary-compatibility.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index 00c63ef575..db825f74d0 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -29,6 +29,7 @@ if [ -z "${PROTOBUF_RUNTIME_VERSION}" ]; then exit 1 fi +# Create a Map of Non-Cloud APIs (Key: Maven Artifact ID Prefix, Value: Maven Group ID) declare -A nonCloudAPIs nonCloudAPIs["grafeas"]="com.google.cloud" nonCloudAPIs["google-maps"]="com.google.maps" @@ -45,17 +46,17 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma # Perform source-compatibility testing on main (latest changes) git clone "https://github.com/googleapis/$repo.git" --depth=1 pushd "$repo" - # Install all modules to ~/.m2 (there can be multiple relevant versions i.e. core, admin, control) + # Install all modules to ~/.m2 (there can be multiple relevant artifacts to test i.e. core, admin, control) mvn -B -ntp clean install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip ARTIFACT_LIST="" # Match all artifacts that start with google-cloud to exclude any proto and grpc modules. # Additionally, exclude any matches to BOM artifacts or emulators - # The artifact list will look something like "com.google.cloud:google-cloud-accessapproval:2.60.0-SNAPSHOT,com.google.cloud:google-cloud-aiplatform:3.60.0-SNAPSHOT," - CLOUD_ARTIFACT_LIST=$(cat "versions.txt" | grep "^google-cloud" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: '{$1="com.google.cloud:"$1; $2=""; print}' OFS=: | sed 's/::/:/' | tr '\n' ',') + # The artifact list will look like "com.google.cloud:google-cloud-accessapproval:2.60.0-SNAPSHOT,com.google.cloud:google-cloud-aiplatform:3.60.0-SNAPSHOT," + CLOUD_ARTIFACT_LIST=$(cat "versions.txt" | grep "^google-cloud" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: "{\$1=\"com.google.cloud:\"\$1; \$2=\"\"; print}" OFS=: | sed 's/::/:/' | tr '\n' ',') + # Remove the trailing comma after the last entry CLOUD_ARTIFACT_LIST=${CLOUD_ARTIFACT_LIST%,} if [ -n "${CLOUD_ARTIFACT_LIST}" ]; then - # Remove the trailing comma after the last entry ARTIFACT_LIST="${ARTIFACT_LIST},${CLOUD_ARTIFACT_LIST}" fi @@ -63,10 +64,11 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma if [ "${repo}" == "google-cloud-java" ]; then for key in "${!nonCloudAPIs[@]}"; do value="${nonCloudAPIs[$key]}" - echo "Key: $key, Value: $value" - NON_CLOUD_ARTIFACT_LIST=$(cat "versions.txt" | grep "^${key}" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: "{\$1=\"${value}:\$1; \$2=\"\"; print}" OFS=: | sed 's/::/:/' | tr '\n' ',') + NON_CLOUD_ARTIFACT_LIST=$(cat "versions.txt" | grep "^${key}" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: "{\$1=\"${value}:\"\$1; \$2=\"\"; print}" OFS=: | sed 's/::/:/' | tr '\n' ',') NON_CLOUD_ARTIFACT_LIST=${NON_CLOUD_ARTIFACT_LIST%,} - ARTIFACT_LIST="${ARTIFACT_LIST},${NON_CLOUD_ARTIFACT_LIST}" + if [ -n "${NON_CLOUD_ARTIFACT_LIST}" ]; then + ARTIFACT_LIST="${ARTIFACT_LIST},${NON_CLOUD_ARTIFACT_LIST}" + fi done fi From 9d125ac5043f3bfac5709fc0d0c50b4c80a8e437 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 24 Feb 2025 15:21:21 -0500 Subject: [PATCH 08/34] ch: Fix CI --- .../downstream_protobuf_compatibility_check_nightly.yaml | 1 + .kokoro/nightly/downstream-protobuf-source-compatibility.sh | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml b/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml index 93b5670237..102b46f300 100644 --- a/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml +++ b/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml @@ -46,6 +46,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 with: + # Use Java 11 for this as Linkage Checker is only compatible with Java 11 or below java-version: 11 distribution: temurin - name: Print Protobuf-Java testing version diff --git a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh index 41af54f81e..c7caeb724c 100755 --- a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh @@ -35,7 +35,8 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma pushd "$repo" # Compile the Handwritten Library with the Protobuf-Java version to test source compatibility - mvn clean compile -B -V -ntp \ + # Run unit tests to help check for any behavior differences (dependant on coverage) + mvn clean test -B -V -ntp \ -Dclirr.skip=true \ -Denforcer.skip=true \ -Dmaven.javadoc.skip=true \ From dbe30517844478de836bb6fc368b639365127887 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 24 Feb 2025 15:59:19 -0500 Subject: [PATCH 09/34] chore: Clean up script --- ...ownstream-protobuf-binary-compatibility.sh | 57 +++++++++---------- 1 file changed, 26 insertions(+), 31 deletions(-) diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index db825f74d0..339c576ff9 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -29,16 +29,17 @@ if [ -z "${PROTOBUF_RUNTIME_VERSION}" ]; then exit 1 fi -# Create a Map of Non-Cloud APIs (Key: Maven Artifact ID Prefix, Value: Maven Group ID) -declare -A nonCloudAPIs -nonCloudAPIs["grafeas"]="com.google.cloud" -nonCloudAPIs["google-maps"]="com.google.maps" -nonCloudAPIs["google-shopping"]="com.google.shopping" +# Create a Map of possible API names (Key: Maven Artifact ID Prefix, Value: Maven Group ID) +declare -A api_maven_map +api_maven_map["google-cloud"]="com.google.cloud" +api_maven_map["grafeas"]="io.grafeas" +api_maven_map["google-maps"]="com.google.maps" +api_maven_map["google-shopping"]="com.google.shopping" # cloud-opensource-java contains the Linkage Checker tool git clone https://github.com/GoogleCloudPlatform/cloud-opensource-java.git pushd cloud-opensource-java -mvn -B -ntp clean compile +mvn -B -ntp clean compile -T 1C # Linkage Checker tool resides in the /dependencies subfolder pushd dependencies @@ -47,38 +48,32 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma git clone "https://github.com/googleapis/$repo.git" --depth=1 pushd "$repo" # Install all modules to ~/.m2 (there can be multiple relevant artifacts to test i.e. core, admin, control) - mvn -B -ntp clean install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip + mvn -B -ntp install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip - ARTIFACT_LIST="" - # Match all artifacts that start with google-cloud to exclude any proto and grpc modules. - # Additionally, exclude any matches to BOM artifacts or emulators - # The artifact list will look like "com.google.cloud:google-cloud-accessapproval:2.60.0-SNAPSHOT,com.google.cloud:google-cloud-aiplatform:3.60.0-SNAPSHOT," - CLOUD_ARTIFACT_LIST=$(cat "versions.txt" | grep "^google-cloud" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: "{\$1=\"com.google.cloud:\"\$1; \$2=\"\"; print}" OFS=: | sed 's/::/:/' | tr '\n' ',') - # Remove the trailing comma after the last entry - CLOUD_ARTIFACT_LIST=${CLOUD_ARTIFACT_LIST%,} - if [ -n "${CLOUD_ARTIFACT_LIST}" ]; then - ARTIFACT_LIST="${ARTIFACT_LIST},${CLOUD_ARTIFACT_LIST}" - fi + artifact_list="" + for artifact_id_prefix in "${!api_maven_map[@]}"; do + group_id="${api_maven_map[${artifact_id_prefix}]}" - # This logic is only for google-cloud-java as there are non-cloud APIs included - if [ "${repo}" == "google-cloud-java" ]; then - for key in "${!nonCloudAPIs[@]}"; do - value="${nonCloudAPIs[$key]}" - NON_CLOUD_ARTIFACT_LIST=$(cat "versions.txt" | grep "^${key}" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: "{\$1=\"${value}:\"\$1; \$2=\"\"; print}" OFS=: | sed 's/::/:/' | tr '\n' ',') - NON_CLOUD_ARTIFACT_LIST=${NON_CLOUD_ARTIFACT_LIST%,} - if [ -n "${NON_CLOUD_ARTIFACT_LIST}" ]; then - ARTIFACT_LIST="${ARTIFACT_LIST},${NON_CLOUD_ARTIFACT_LIST}" - fi - done - fi + # Match all artifacts that start with google-cloud to exclude any proto and grpc modules. + # Additionally, exclude any matches to BOM artifacts or emulators + # The artifact list will look like "com.google.cloud:google-cloud-accessapproval:2.60.0-SNAPSHOT,com.google.cloud:google-cloud-aiplatform:3.60.0-SNAPSHOT," + repo_artifact_list=$(cat "versions.txt" | grep "^${artifact_id_prefix}" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: "{\$1=\"${group_id}:\"\$1; \$2=\"\"; print}" OFS=: | sed 's/::/:/' | tr '\n' ',') + + # Only google-cloud-java has non-cloud APIs. Only add if there are artifacts to be added + if [ -n "${repo_artifact_list}" ]; then + # Remove the trailing comma after the last entry + repo_artifact_list=${repo_artifact_list%,} + artifact_list="${artifact_list},${repo_artifact_list}" + fi + done # Linkage Checker /dependencies popd - if [ -n "${ARTIFACT_LIST}" ]; then - echo "Found artifacts ${ARTIFACT_LIST}" + if [ -n "${artifact_list}" ]; then + echo "Found artifacts ${artifact_list}" # The `-s` argument filters the linkage check problems that stem from the artifact - program_args="-r --artifacts ${ARTIFACT_LIST},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${ARTIFACT_LIST}" + program_args="-r --artifacts ${artifact_list},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${artifact_list}" echo "Linkage Checker Program Arguments: ${program_args}" mvn -B -ntp exec:java -Dexec.mainClass="com.google.cloud.tools.opensource.classpath.LinkageCheckerMain" -Dexec.args="${program_args}" fi From fe491b61bc549afc6ae46a5b3bc0c57fcc6fec53 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 24 Feb 2025 16:19:25 -0500 Subject: [PATCH 10/34] chore: Clean up script --- ...ownstream-protobuf-binary-compatibility.sh | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index 339c576ff9..dd9f8a294f 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -30,6 +30,7 @@ if [ -z "${PROTOBUF_RUNTIME_VERSION}" ]; then fi # Create a Map of possible API names (Key: Maven Artifact ID Prefix, Value: Maven Group ID) +# Non-Cloud APIs have a different Group ID declare -A api_maven_map api_maven_map["google-cloud"]="com.google.cloud" api_maven_map["grafeas"]="io.grafeas" @@ -54,17 +55,19 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma for artifact_id_prefix in "${!api_maven_map[@]}"; do group_id="${api_maven_map[${artifact_id_prefix}]}" - # Match all artifacts that start with google-cloud to exclude any proto and grpc modules. - # Additionally, exclude any matches to BOM artifacts or emulators - # The artifact list will look like "com.google.cloud:google-cloud-accessapproval:2.60.0-SNAPSHOT,com.google.cloud:google-cloud-aiplatform:3.60.0-SNAPSHOT," - repo_artifact_list=$(cat "versions.txt" | grep "^${artifact_id_prefix}" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: "{\$1=\"${group_id}:\"\$1; \$2=\"\"; print}" OFS=: | sed 's/::/:/' | tr '\n' ',') - - # Only google-cloud-java has non-cloud APIs. Only add if there are artifacts to be added - if [ -n "${repo_artifact_list}" ]; then - # Remove the trailing comma after the last entry - repo_artifact_list=${repo_artifact_list%,} - artifact_list="${artifact_list},${repo_artifact_list}" + # Match all artifacts that start with the artifact_id_prefix to exclude any proto and grpc modules. + repo_artifact_list=$(cat "versions.txt" | grep "^${artifact_id_prefix}" || true) + # If there are no matching artifacts, then skip. Only google-cloud-java has non-cloud APIs + if [ -z "${repo_artifact_list}" ]; then + continue fi + + # Exclude any matches to BOM artifacts or emulators. The repo artifact list will look like: + # "com.google.cloud:google-cloud-accessapproval:2.60.0-SNAPSHOT,com.google.cloud:google-cloud-aiplatform:3.60.0-SNAPSHOT," + repo_artifact_list=$(echo "${repo_artifact_list}" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: "{\$1=\"${group_id}:\"\$1; \$2=\"\"; print}" OFS=: | sed 's/::/:/' | tr '\n' ',') + # Remove the trailing comma after the last entry + repo_artifact_list=${repo_artifact_list%,} + artifact_list="${artifact_list},${repo_artifact_list}" done # Linkage Checker /dependencies From 002868d5c5f34192355d23ce384233d59ea38f15 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 24 Feb 2025 16:20:59 -0500 Subject: [PATCH 11/34] chore: Clean up script --- .kokoro/nightly/downstream-protobuf-binary-compatibility.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index dd9f8a294f..2c8b54e6c5 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -45,10 +45,10 @@ mvn -B -ntp clean compile -T 1C pushd dependencies for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma - # Perform source-compatibility testing on main (latest changes) + # Perform binary-compatibility testing on main (latest changes) and do not pull history git clone "https://github.com/googleapis/$repo.git" --depth=1 pushd "$repo" - # Install all modules to ~/.m2 (there can be multiple relevant artifacts to test i.e. core, admin, control) + # Install all repo modules to ~/.m2 (there can be multiple relevant artifacts to test i.e. core, admin, control) mvn -B -ntp install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip artifact_list="" From c20ca056c3f39d317bb52a08a2e9ba32730a41d3 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 24 Feb 2025 16:46:03 -0500 Subject: [PATCH 12/34] chore: Clean up script --- .../downstream-protobuf-binary-compatibility.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index 2c8b54e6c5..0bd5cb1766 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -45,19 +45,20 @@ mvn -B -ntp clean compile -T 1C pushd dependencies for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma - # Perform binary-compatibility testing on main (latest changes) and do not pull history + # Perform testing on main (with latest changes). Shallow copy as history is not important git clone "https://github.com/googleapis/$repo.git" --depth=1 pushd "$repo" # Install all repo modules to ~/.m2 (there can be multiple relevant artifacts to test i.e. core, admin, control) mvn -B -ntp install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip + # The artifact_list will be a comma separate list of artifacts artifact_list="" for artifact_id_prefix in "${!api_maven_map[@]}"; do group_id="${api_maven_map[${artifact_id_prefix}]}" # Match all artifacts that start with the artifact_id_prefix to exclude any proto and grpc modules. repo_artifact_list=$(cat "versions.txt" | grep "^${artifact_id_prefix}" || true) - # If there are no matching artifacts, then skip. Only google-cloud-java has non-cloud APIs + # Only google-cloud-java has non-cloud APIs. If there are no matching artifacts, then skip. if [ -z "${repo_artifact_list}" ]; then continue fi @@ -67,12 +68,19 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma repo_artifact_list=$(echo "${repo_artifact_list}" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: "{\$1=\"${group_id}:\"\$1; \$2=\"\"; print}" OFS=: | sed 's/::/:/' | tr '\n' ',') # Remove the trailing comma after the last entry repo_artifact_list=${repo_artifact_list%,} - artifact_list="${artifact_list},${repo_artifact_list}" + + # The first entry added is not separated with a comma. Avoids generating `,{ARTIFACT_LIST}` + if [ -z "${artifact_list}" ]; then + artifact_list="${repo_artifact_list}" + else + artifact_list="${artifact_list},${repo_artifact_list}" + fi done # Linkage Checker /dependencies popd + # Only run Linkage Checker if the repo has any relevant artifacts to test for if [ -n "${artifact_list}" ]; then echo "Found artifacts ${artifact_list}" # The `-s` argument filters the linkage check problems that stem from the artifact From b1b4d217ed5fdec09db0dc478ca9543a0dc1b778 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Thu, 6 Mar 2025 15:16:37 -0500 Subject: [PATCH 13/34] chore: Add mappings for the handwritten libraries --- ...ownstream-protobuf-binary-compatibility.sh | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index 0bd5cb1766..bfda2286df 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -29,13 +29,17 @@ if [ -z "${PROTOBUF_RUNTIME_VERSION}" ]; then exit 1 fi -# Create a Map of possible API names (Key: Maven Artifact ID Prefix, Value: Maven Group ID) -# Non-Cloud APIs have a different Group ID -declare -A api_maven_map -api_maven_map["google-cloud"]="com.google.cloud" -api_maven_map["grafeas"]="io.grafeas" -api_maven_map["google-maps"]="com.google.maps" -api_maven_map["google-shopping"]="com.google.shopping" +# Create two mappings of possible API names (Key: Maven Artifact ID Prefix, Value: Maven Group ID) +# for the libraries that should be tested. +# 1. These are special handwritten libraries in google-cloud-java that should be tested +declare -A google_cloud_java_handwritten_libraries +google_cloud_java_handwritten_libraries["grafeas"]="io.grafeas" +google_cloud_java_handwritten_libraries["vertexai"]="com.google.cloud" +google_cloud_java_handwritten_libraries["resourcemanager"]="com.google.cloud" + +# 2. These are the mappings of all the downstream handwritten libraries +declare -A gcp_handwritten_libraries +gcp_handwritten_libraries["google-cloud"]="com.google.cloud" # cloud-opensource-java contains the Linkage Checker tool git clone https://github.com/GoogleCloudPlatform/cloud-opensource-java.git @@ -53,6 +57,13 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma # The artifact_list will be a comma separate list of artifacts artifact_list="" + + if [ "${repo}" == "google-cloud-java" ]; then + api_maven_map=google_cloud_java_handwritten_libraries + else + api_maven_map=gcp_handwritten_libraries + fi + for artifact_id_prefix in "${!api_maven_map[@]}"; do group_id="${api_maven_map[${artifact_id_prefix}]}" From 8a1aa88d84192da119a8e7bd3adf2d9197397800 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Thu, 6 Mar 2025 15:41:14 -0500 Subject: [PATCH 14/34] chore: Fix typo --- .kokoro/nightly/downstream-protobuf-binary-compatibility.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index bfda2286df..c6a73f2ba7 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -17,14 +17,14 @@ set -eo pipefail # Comma-delimited list of repos to test with the local java-shared-dependencies if [ -z "${REPOS_UNDER_TEST}" ]; then - echo "REPOS_UNDER_TEST must be set to run downstream-protobuf-source-compatibility.sh" + echo "REPOS_UNDER_TEST must be set to run downstream-protobuf-binary-compatibility.sh" echo "Expects a comma-delimited list: i.e REPOS_UNDER_TEST=\"java-bigtable,java-bigquery\"" exit 1 fi # Version of Protobuf-Java runtime to compile with if [ -z "${PROTOBUF_RUNTIME_VERSION}" ]; then - echo "PROTOBUF_RUNTIME_VERSION must be set to run downstream-protobuf-source-compatibility.sh" + echo "PROTOBUF_RUNTIME_VERSION must be set to run downstream-protobuf-binary-compatibility.sh" echo "Expects a single Protobuf-Java runtime version i.e. PROTOBUF_RUNTIME_VERSION=\"4.28.3\"" exit 1 fi From fa542c75b1521131d3010b6d6a57521751e9959c Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Thu, 6 Mar 2025 17:30:21 -0500 Subject: [PATCH 15/34] chore: Skip installing if repo is already built --- ..._protobuf_compatibility_check_nightly.yaml | 37 ++++++---- ...ownstream-protobuf-binary-compatibility.sh | 70 +++++++++++-------- ...ownstream-protobuf-source-compatibility.sh | 13 ++-- 3 files changed, 72 insertions(+), 48 deletions(-) diff --git a/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml b/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml index 102b46f300..832b370209 100644 --- a/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml +++ b/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml @@ -25,34 +25,41 @@ jobs: fail-fast: false matrix: repo: - - google-cloud-java +# - google-cloud-java - java-bigtable - - java-bigquery - - java-bigquerystorage - - java-datastore - - java-firestore - - java-logging - - java-logging-logback - - java-pubsub - - java-pubsublite - - java-spanner-jdbc - - java-spanner - - java-storage - - java-storage-nio +# - java-bigquery +# - java-bigquerystorage +# - java-datastore +# - java-firestore +# - java-logging +# - java-logging-logback +# - java-pubsub +# - java-pubsublite +# - java-spanner-jdbc +# - java-spanner +# - java-storage +# - java-storage-nio # Default Protobuf-Java versions to use are specified here. Without this, the nightly workflow won't know # which values to use and would resolve to ''. protobuf-version: ${{ fromJSON(format('[{0}]', inputs.protobuf_runtime_versions || '"3.25.5","4.28.3"')) }} steps: - uses: actions/checkout@v4 + with: + path: sdk-platform-java - uses: actions/setup-java@v4 with: # Use Java 11 for this as Linkage Checker is only compatible with Java 11 or below java-version: 11 distribution: temurin + - uses: actions/checkout@v4 + with: + repository: "googleapis/${{ matrix.repo }}" + path: "${{ matrix.repo }}" + - run: mvn -B -ntp install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip - name: Print Protobuf-Java testing version run: echo "Testing with Protobuf-Java v${{ matrix.protobuf-version }}" - name: Perform downstream source compatibility testing - run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version}}" ./.kokoro/nightly/downstream-protobuf-source-compatibility.sh + run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version}}" REPOS_INSTALLED="true" ./.kokoro/nightly/downstream-protobuf-source-compatibility.sh - name: Perform downstream binary compatibility testing - run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version}}" ./.kokoro/nightly/downstream-protobuf-binary-compatibility.sh + run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version}}" REPOS_INSTALLED="true" ./.kokoro/nightly/downstream-protobuf-binary-compatibility.sh diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index c6a73f2ba7..30883c0c16 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -15,6 +15,8 @@ set -eo pipefail +pwd + # Comma-delimited list of repos to test with the local java-shared-dependencies if [ -z "${REPOS_UNDER_TEST}" ]; then echo "REPOS_UNDER_TEST must be set to run downstream-protobuf-binary-compatibility.sh" @@ -34,8 +36,8 @@ fi # 1. These are special handwritten libraries in google-cloud-java that should be tested declare -A google_cloud_java_handwritten_libraries google_cloud_java_handwritten_libraries["grafeas"]="io.grafeas" -google_cloud_java_handwritten_libraries["vertexai"]="com.google.cloud" -google_cloud_java_handwritten_libraries["resourcemanager"]="com.google.cloud" +google_cloud_java_handwritten_libraries["google-cloud-vertexai"]="com.google.cloud" +google_cloud_java_handwritten_libraries["google-cloud-resourcemanager"]="com.google.cloud" # 2. These are the mappings of all the downstream handwritten libraries declare -A gcp_handwritten_libraries @@ -48,32 +50,14 @@ mvn -B -ntp clean compile -T 1C # Linkage Checker tool resides in the /dependencies subfolder pushd dependencies -for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma - # Perform testing on main (with latest changes). Shallow copy as history is not important - git clone "https://github.com/googleapis/$repo.git" --depth=1 - pushd "$repo" - # Install all repo modules to ~/.m2 (there can be multiple relevant artifacts to test i.e. core, admin, control) - mvn -B -ntp install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip - - # The artifact_list will be a comma separate list of artifacts - artifact_list="" - - if [ "${repo}" == "google-cloud-java" ]; then - api_maven_map=google_cloud_java_handwritten_libraries - else - api_maven_map=gcp_handwritten_libraries - fi - - for artifact_id_prefix in "${!api_maven_map[@]}"; do - group_id="${api_maven_map[${artifact_id_prefix}]}" - - # Match all artifacts that start with the artifact_id_prefix to exclude any proto and grpc modules. - repo_artifact_list=$(cat "versions.txt" | grep "^${artifact_id_prefix}" || true) - # Only google-cloud-java has non-cloud APIs. If there are no matching artifacts, then skip. - if [ -z "${repo_artifact_list}" ]; then - continue - fi +function build_artifact_list() { + local artifact_id_prefix=$1 + local group_id=$2 + # Match all artifacts that start with the artifact_id_prefix to exclude any proto and grpc modules. + repo_artifact_list=$(cat "versions.txt" | grep "^${artifact_id_prefix}" || true) + # Only proceed if there are matching elements + if [ -n "${repo_artifact_list}" ]; then # Exclude any matches to BOM artifacts or emulators. The repo artifact list will look like: # "com.google.cloud:google-cloud-accessapproval:2.60.0-SNAPSHOT,com.google.cloud:google-cloud-aiplatform:3.60.0-SNAPSHOT," repo_artifact_list=$(echo "${repo_artifact_list}" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: "{\$1=\"${group_id}:\"\$1; \$2=\"\"; print}" OFS=: | sed 's/::/:/' | tr '\n' ',') @@ -86,11 +70,40 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma else artifact_list="${artifact_list},${repo_artifact_list}" fi - done + fi +} + +for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma + if [ "${REPOS_INSTALLED}" != "true" ]; then + # Perform testing on main (with latest changes). Shallow copy as history is not important + git clone "https://github.com/googleapis/${repo}.git" --depth=1 + pushd "${repo}" + # Install all repo modules to ~/.m2 (there can be multiple relevant artifacts to test i.e. core, admin, control) + mvn -B -ntp install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip + else + mkdir "${repo}" + pushd "${repo}" + curl -O "https://raw.githubusercontent.com/googleapis/${repo}/refs/heads/main/versions.txt" + fi + + # The artifact_list will be a comma separate list of artifacts + artifact_list="" + if [ "${repo}" == "google-cloud-java" ]; then + for artifact_id_prefix in "${!google_cloud_java_handwritten_libraries[@]}"; do + group_id="${google_cloud_java_handwritten_libraries[${artifact_id_prefix}]}" + build_artifact_list "${artifact_id_prefix}" "${group_id}" + done + else + for artifact_id_prefix in "${!gcp_handwritten_libraries[@]}"; do + group_id="${gcp_handwritten_libraries[${artifact_id_prefix}]}" + build_artifact_list "${artifact_id_prefix}" "${group_id}" + done + fi # Linkage Checker /dependencies popd + echo "Artifact List: ${artifact_list}" # Only run Linkage Checker if the repo has any relevant artifacts to test for if [ -n "${artifact_list}" ]; then echo "Found artifacts ${artifact_list}" @@ -99,6 +112,7 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma echo "Linkage Checker Program Arguments: ${program_args}" mvn -B -ntp exec:java -Dexec.mainClass="com.google.cloud.tools.opensource.classpath.LinkageCheckerMain" -Dexec.args="${program_args}" fi + echo "done" done popd popd diff --git a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh index c7caeb724c..8be8889482 100755 --- a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh @@ -15,6 +15,8 @@ set -eo pipefail +pwd + # Comma-delimited list of repos to test with the local java-shared-dependencies if [ -z "${REPOS_UNDER_TEST}" ]; then echo "REPOS_UNDER_TEST must be set to run downstream-protobuf-source-compatibility.sh" @@ -30,17 +32,18 @@ if [ -z "${PROTOBUF_RUNTIME_VERSION}" ]; then fi for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma - # Perform source-compatibility testing on main (latest changes) - git clone "https://github.com/googleapis/$repo.git" --depth=1 - pushd "$repo" + if [ "${REPOS_INSTALLED}" != "true" ]; then + # Perform source-compatibility testing on main (latest changes) + git clone "https://github.com/googleapis/$repo.git" --depth=1 + pushd "$repo" + fi # Compile the Handwritten Library with the Protobuf-Java version to test source compatibility # Run unit tests to help check for any behavior differences (dependant on coverage) - mvn clean test -B -V -ntp \ + mvn test -B -V -ntp \ -Dclirr.skip=true \ -Denforcer.skip=true \ -Dmaven.javadoc.skip=true \ -Dprotobuf.version=${PROTOBUF_RUNTIME_VERSION} \ -T 1C - popd done \ No newline at end of file From 7f295aa36490ba4f560b56944e59f43f0571f13d Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Thu, 6 Mar 2025 17:36:28 -0500 Subject: [PATCH 16/34] ci: cd to downstream repo and build it --- .../downstream_protobuf_compatibility_check_nightly.yaml | 4 +++- .kokoro/nightly/downstream-protobuf-binary-compatibility.sh | 1 - .kokoro/nightly/downstream-protobuf-source-compatibility.sh | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml b/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml index 832b370209..4ebadc8730 100644 --- a/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml +++ b/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml @@ -55,7 +55,9 @@ jobs: with: repository: "googleapis/${{ matrix.repo }}" path: "${{ matrix.repo }}" - - run: mvn -B -ntp install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip + - run: | + cd "${{ matrix.repo }}" + mvn -B -ntp install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip - name: Print Protobuf-Java testing version run: echo "Testing with Protobuf-Java v${{ matrix.protobuf-version }}" - name: Perform downstream source compatibility testing diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index 30883c0c16..70995c8120 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -81,7 +81,6 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma # Install all repo modules to ~/.m2 (there can be multiple relevant artifacts to test i.e. core, admin, control) mvn -B -ntp install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip else - mkdir "${repo}" pushd "${repo}" curl -O "https://raw.githubusercontent.com/googleapis/${repo}/refs/heads/main/versions.txt" fi diff --git a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh index 8be8889482..1fc651608a 100755 --- a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh @@ -35,9 +35,10 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma if [ "${REPOS_INSTALLED}" != "true" ]; then # Perform source-compatibility testing on main (latest changes) git clone "https://github.com/googleapis/$repo.git" --depth=1 - pushd "$repo" fi + pushd "$repo" + # Compile the Handwritten Library with the Protobuf-Java version to test source compatibility # Run unit tests to help check for any behavior differences (dependant on coverage) mvn test -B -V -ntp \ From 5237b8eb3fd94ac14ffd532a8d8f03e587b75806 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Thu, 6 Mar 2025 17:40:10 -0500 Subject: [PATCH 17/34] ci: cd back into root directory --- .../downstream_protobuf_compatibility_check_nightly.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml b/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml index 4ebadc8730..03381b951d 100644 --- a/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml +++ b/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml @@ -56,8 +56,12 @@ jobs: repository: "googleapis/${{ matrix.repo }}" path: "${{ matrix.repo }}" - run: | - cd "${{ matrix.repo }}" + pwd + pushd "${{ matrix.repo }}" mvn -B -ntp install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip + pwd + popd + pwd - name: Print Protobuf-Java testing version run: echo "Testing with Protobuf-Java v${{ matrix.protobuf-version }}" - name: Perform downstream source compatibility testing From c7a4b065a4b1c26c9ef248b1b4cdf02e91f56d9c Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Thu, 6 Mar 2025 17:53:05 -0500 Subject: [PATCH 18/34] ci: use nested directories --- .../downstream_protobuf_compatibility_check_nightly.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml b/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml index 03381b951d..d6f6a28ef3 100644 --- a/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml +++ b/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml @@ -43,18 +43,16 @@ jobs: # which values to use and would resolve to ''. protobuf-version: ${{ fromJSON(format('[{0}]', inputs.protobuf_runtime_versions || '"3.25.5","4.28.3"')) }} steps: + - uses: actions/checkout@v4 - uses: actions/checkout@v4 with: - path: sdk-platform-java + repository: "googleapis/${{ matrix.repo }}" + path: "${{ matrix.repo }}" - uses: actions/setup-java@v4 with: # Use Java 11 for this as Linkage Checker is only compatible with Java 11 or below java-version: 11 distribution: temurin - - uses: actions/checkout@v4 - with: - repository: "googleapis/${{ matrix.repo }}" - path: "${{ matrix.repo }}" - run: | pwd pushd "${{ matrix.repo }}" From 1c07195bdb956bddf745d839b5876a29a6128f4b Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Thu, 6 Mar 2025 18:02:43 -0500 Subject: [PATCH 19/34] ci: use nested directories --- .../downstream_protobuf_compatibility_check_nightly.yaml | 4 ++-- .../nightly/downstream-protobuf-binary-compatibility.sh | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml b/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml index d6f6a28ef3..de69196319 100644 --- a/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml +++ b/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml @@ -63,7 +63,7 @@ jobs: - name: Print Protobuf-Java testing version run: echo "Testing with Protobuf-Java v${{ matrix.protobuf-version }}" - name: Perform downstream source compatibility testing - run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version}}" REPOS_INSTALLED="true" ./.kokoro/nightly/downstream-protobuf-source-compatibility.sh + run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version}}" REPOS_INSTALLED_LOCALLY="true" ./.kokoro/nightly/downstream-protobuf-source-compatibility.sh - name: Perform downstream binary compatibility testing - run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version}}" REPOS_INSTALLED="true" ./.kokoro/nightly/downstream-protobuf-binary-compatibility.sh + run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version}}" REPOS_INSTALLED_LOCALLY="true" ./.kokoro/nightly/downstream-protobuf-binary-compatibility.sh diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index 70995c8120..5554908cad 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -74,15 +74,15 @@ function build_artifact_list() { } for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma - if [ "${REPOS_INSTALLED}" != "true" ]; then + if [ "${REPOS_INSTALLED_LOCALLY}" != "true" ]; then # Perform testing on main (with latest changes). Shallow copy as history is not important git clone "https://github.com/googleapis/${repo}.git" --depth=1 - pushd "${repo}" + pushd "../../${repo}" # Install all repo modules to ~/.m2 (there can be multiple relevant artifacts to test i.e. core, admin, control) mvn -B -ntp install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip else - pushd "${repo}" - curl -O "https://raw.githubusercontent.com/googleapis/${repo}/refs/heads/main/versions.txt" + pushd "../../${repo}" +# curl -O "https://raw.githubusercontent.com/googleapis/${repo}/refs/heads/main/versions.txt" fi # The artifact_list will be a comma separate list of artifacts From 49f658cfa8e621ec02f03536c44fd788517bb4e4 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Thu, 6 Mar 2025 18:05:13 -0500 Subject: [PATCH 20/34] ci: use nested directories --- .../downstream_protobuf_compatibility_check_nightly.yaml | 4 ++-- .kokoro/nightly/downstream-protobuf-source-compatibility.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml b/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml index de69196319..0e21cd22bf 100644 --- a/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml +++ b/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml @@ -63,7 +63,7 @@ jobs: - name: Print Protobuf-Java testing version run: echo "Testing with Protobuf-Java v${{ matrix.protobuf-version }}" - name: Perform downstream source compatibility testing - run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version}}" REPOS_INSTALLED_LOCALLY="true" ./.kokoro/nightly/downstream-protobuf-source-compatibility.sh + run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version }}" REPOS_INSTALLED_LOCALLY="true" ./.kokoro/nightly/downstream-protobuf-source-compatibility.sh - name: Perform downstream binary compatibility testing - run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version}}" REPOS_INSTALLED_LOCALLY="true" ./.kokoro/nightly/downstream-protobuf-binary-compatibility.sh + run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version }}" REPOS_INSTALLED_LOCALLY="true" ./.kokoro/nightly/downstream-protobuf-binary-compatibility.sh diff --git a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh index 1fc651608a..6cac62d8a6 100755 --- a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh @@ -32,7 +32,7 @@ if [ -z "${PROTOBUF_RUNTIME_VERSION}" ]; then fi for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma - if [ "${REPOS_INSTALLED}" != "true" ]; then + if [ "${REPOS_INSTALLED_LOCALLY}" != "true" ]; then # Perform source-compatibility testing on main (latest changes) git clone "https://github.com/googleapis/$repo.git" --depth=1 fi From dffa8fa66bed53372ad6d96c0c20cef606af7fef Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Thu, 6 Mar 2025 18:11:38 -0500 Subject: [PATCH 21/34] ci: Re-enable this for all handwritten libraries --- ..._protobuf_compatibility_check_nightly.yaml | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml b/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml index 0e21cd22bf..5959afd1d9 100644 --- a/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml +++ b/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml @@ -25,26 +25,28 @@ jobs: fail-fast: false matrix: repo: -# - google-cloud-java + - google-cloud-java - java-bigtable -# - java-bigquery -# - java-bigquerystorage -# - java-datastore -# - java-firestore -# - java-logging -# - java-logging-logback -# - java-pubsub -# - java-pubsublite -# - java-spanner-jdbc -# - java-spanner -# - java-storage -# - java-storage-nio + - java-bigquery + - java-bigquerystorage + - java-datastore + - java-firestore + - java-logging + - java-logging-logback + - java-pubsub + - java-pubsublite + - java-spanner-jdbc + - java-spanner + - java-storage + - java-storage-nio # Default Protobuf-Java versions to use are specified here. Without this, the nightly workflow won't know # which values to use and would resolve to ''. protobuf-version: ${{ fromJSON(format('[{0}]', inputs.protobuf_runtime_versions || '"3.25.5","4.28.3"')) }} steps: - - uses: actions/checkout@v4 - - uses: actions/checkout@v4 + - name: Checkout sdk-platform-java repo + uses: actions/checkout@v4 + - name: Checkout downstream handwritten repo + uses: actions/checkout@v4 with: repository: "googleapis/${{ matrix.repo }}" path: "${{ matrix.repo }}" @@ -53,13 +55,11 @@ jobs: # Use Java 11 for this as Linkage Checker is only compatible with Java 11 or below java-version: 11 distribution: temurin - - run: | - pwd + - name: Install downstream repo locally + run: | pushd "${{ matrix.repo }}" mvn -B -ntp install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip - pwd popd - pwd - name: Print Protobuf-Java testing version run: echo "Testing with Protobuf-Java v${{ matrix.protobuf-version }}" - name: Perform downstream source compatibility testing From edde57da6e951482a1979d922aebfe62a0fbe0c9 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Fri, 7 Mar 2025 14:54:23 -0500 Subject: [PATCH 22/34] chore: Use function to build artifact list --- ...ownstream-protobuf-binary-compatibility.sh | 85 +++++++++---------- ...ownstream-protobuf-source-compatibility.sh | 2 +- 2 files changed, 42 insertions(+), 45 deletions(-) diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index 5554908cad..c6ad040de2 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -15,8 +15,6 @@ set -eo pipefail -pwd - # Comma-delimited list of repos to test with the local java-shared-dependencies if [ -z "${REPOS_UNDER_TEST}" ]; then echo "REPOS_UNDER_TEST must be set to run downstream-protobuf-binary-compatibility.sh" @@ -34,14 +32,42 @@ fi # Create two mappings of possible API names (Key: Maven Artifact ID Prefix, Value: Maven Group ID) # for the libraries that should be tested. # 1. These are special handwritten libraries in google-cloud-java that should be tested -declare -A google_cloud_java_handwritten_libraries -google_cloud_java_handwritten_libraries["grafeas"]="io.grafeas" -google_cloud_java_handwritten_libraries["google-cloud-vertexai"]="com.google.cloud" -google_cloud_java_handwritten_libraries["google-cloud-resourcemanager"]="com.google.cloud" +declare -A monorepo_handwritten_libraries +monorepo_handwritten_libraries["grafeas"]="io.grafeas" +monorepo_handwritten_libraries["google-cloud-vertexai"]="com.google.cloud" +monorepo_handwritten_libraries["google-cloud-resourcemanager"]="com.google.cloud" + +# 2. These are the mappings of all the downstream handwritten libraries' artifacts +declare -A downstream_handwritten_libraries +downstream_handwritten_libraries["google-cloud"]="com.google.cloud" + +# Builds a string output to `artifact_list`. It contains a comma separate list of Maven GAV coordinates. Parses +# the `versions.txt` file by searching for the matching artifact_id_prefix to get the corresponding version. +function build_artifact_list() { + local -n api_maven_mapping=$1 + for artifact_id_prefix in "${!api_maven_mapping[@]}"; do + group_id="${api_maven_mapping[${artifact_id_prefix}]}" -# 2. These are the mappings of all the downstream handwritten libraries -declare -A gcp_handwritten_libraries -gcp_handwritten_libraries["google-cloud"]="com.google.cloud" + # Match all artifacts that start with the artifact_id_prefix to exclude any proto and grpc modules. + repo_artifact_list=$(cat "versions.txt" | grep "^${artifact_id_prefix}" || true) + + # Only proceed if there are matching elements + if [ -n "${repo_artifact_list}" ]; then + # Exclude any matches to BOM artifacts or emulators. The repo artifact list will look like: + # "com.google.cloud:google-cloud-accessapproval:2.60.0-SNAPSHOT,com.google.cloud:google-cloud-aiplatform:3.60.0-SNAPSHOT," + repo_artifact_list=$(echo "${repo_artifact_list}" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: "{\$1=\"${group_id}:\"\$1; \$2=\"\"; print}" OFS=: | sed 's/::/:/' | tr '\n' ',') + # Remove the trailing comma after the last entry + repo_artifact_list=${repo_artifact_list%,} + + # The first entry added is not separated with a comma. Avoids generating `,{ARTIFACT_LIST}` + if [ -z "${artifact_list}" ]; then + artifact_list="${repo_artifact_list}" + else + artifact_list="${artifact_list},${repo_artifact_list}" + fi + fi + done +} # cloud-opensource-java contains the Linkage Checker tool git clone https://github.com/GoogleCloudPlatform/cloud-opensource-java.git @@ -50,53 +76,25 @@ mvn -B -ntp clean compile -T 1C # Linkage Checker tool resides in the /dependencies subfolder pushd dependencies -function build_artifact_list() { - local artifact_id_prefix=$1 - local group_id=$2 - # Match all artifacts that start with the artifact_id_prefix to exclude any proto and grpc modules. - repo_artifact_list=$(cat "versions.txt" | grep "^${artifact_id_prefix}" || true) - - # Only proceed if there are matching elements - if [ -n "${repo_artifact_list}" ]; then - # Exclude any matches to BOM artifacts or emulators. The repo artifact list will look like: - # "com.google.cloud:google-cloud-accessapproval:2.60.0-SNAPSHOT,com.google.cloud:google-cloud-aiplatform:3.60.0-SNAPSHOT," - repo_artifact_list=$(echo "${repo_artifact_list}" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: "{\$1=\"${group_id}:\"\$1; \$2=\"\"; print}" OFS=: | sed 's/::/:/' | tr '\n' ',') - # Remove the trailing comma after the last entry - repo_artifact_list=${repo_artifact_list%,} - - # The first entry added is not separated with a comma. Avoids generating `,{ARTIFACT_LIST}` - if [ -z "${artifact_list}" ]; then - artifact_list="${repo_artifact_list}" - else - artifact_list="${artifact_list},${repo_artifact_list}" - fi - fi -} - for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma if [ "${REPOS_INSTALLED_LOCALLY}" != "true" ]; then # Perform testing on main (with latest changes). Shallow copy as history is not important git clone "https://github.com/googleapis/${repo}.git" --depth=1 - pushd "../../${repo}" + pushd "${repo}" # Install all repo modules to ~/.m2 (there can be multiple relevant artifacts to test i.e. core, admin, control) mvn -B -ntp install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip else + # If installed locally to .m2, then just pull the versions.txt file to parse + mkdir -p "../../${repo}" pushd "../../${repo}" -# curl -O "https://raw.githubusercontent.com/googleapis/${repo}/refs/heads/main/versions.txt" + curl -Os "https://raw.githubusercontent.com/googleapis/${repo}/refs/heads/main/versions.txt" fi - # The artifact_list will be a comma separate list of artifacts artifact_list="" if [ "${repo}" == "google-cloud-java" ]; then - for artifact_id_prefix in "${!google_cloud_java_handwritten_libraries[@]}"; do - group_id="${google_cloud_java_handwritten_libraries[${artifact_id_prefix}]}" - build_artifact_list "${artifact_id_prefix}" "${group_id}" - done + build_artifact_list monorepo_handwritten_libraries else - for artifact_id_prefix in "${!gcp_handwritten_libraries[@]}"; do - group_id="${gcp_handwritten_libraries[${artifact_id_prefix}]}" - build_artifact_list "${artifact_id_prefix}" "${group_id}" - done + build_artifact_list downstream_handwritten_libraries fi # Linkage Checker /dependencies @@ -105,7 +103,6 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma echo "Artifact List: ${artifact_list}" # Only run Linkage Checker if the repo has any relevant artifacts to test for if [ -n "${artifact_list}" ]; then - echo "Found artifacts ${artifact_list}" # The `-s` argument filters the linkage check problems that stem from the artifact program_args="-r --artifacts ${artifact_list},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${artifact_list}" echo "Linkage Checker Program Arguments: ${program_args}" diff --git a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh index 6cac62d8a6..a622ee6a94 100755 --- a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh @@ -41,7 +41,7 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma # Compile the Handwritten Library with the Protobuf-Java version to test source compatibility # Run unit tests to help check for any behavior differences (dependant on coverage) - mvn test -B -V -ntp \ + mvn compile -B -V -ntp \ -Dclirr.skip=true \ -Denforcer.skip=true \ -Dmaven.javadoc.skip=true \ From 594e5a8f451706ac3363bb20c4a5dfc855812076 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Fri, 7 Mar 2025 15:11:27 -0500 Subject: [PATCH 23/34] chore: Use exec-linkage-checker maven profile --- .../downstream-protobuf-binary-compatibility.sh | 10 +++++++++- .../downstream-protobuf-source-compatibility.sh | 8 +++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index c6ad040de2..834c7a2f24 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -15,6 +15,14 @@ set -eo pipefail +# There are three Env Vars that this script uses. +# 1. (Required) REPOS_UNDER_TEST: Comma separate list of the repo names +# 2. (Required) PROTOBUF_RUNTIME_VERSION: Protobuf runtime version to test again +# 3. REPOS_INSTALLED_LOCALLY: Flag (if set to "true) will determine if the repo +# needs to be cloned from github and re-installed locally to the m2. If the artifact +# is already installed locally, this saves time as the artifact is not again compiled +# and built. + # Comma-delimited list of repos to test with the local java-shared-dependencies if [ -z "${REPOS_UNDER_TEST}" ]; then echo "REPOS_UNDER_TEST must be set to run downstream-protobuf-binary-compatibility.sh" @@ -106,7 +114,7 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma # The `-s` argument filters the linkage check problems that stem from the artifact program_args="-r --artifacts ${artifact_list},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${artifact_list}" echo "Linkage Checker Program Arguments: ${program_args}" - mvn -B -ntp exec:java -Dexec.mainClass="com.google.cloud.tools.opensource.classpath.LinkageCheckerMain" -Dexec.args="${program_args}" + mvn -B -ntp exec:java -Dexec.mainClass="com.google.cloud.tools.opensource.classpath.LinkageCheckerMain" -Dexec.args="${program_args}" -P exec-linkage-checker fi echo "done" done diff --git a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh index a622ee6a94..7e0dca16b5 100755 --- a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh @@ -15,7 +15,13 @@ set -eo pipefail -pwd +# There are three Env Vars that this script uses. +# 1. (Required) REPOS_UNDER_TEST: Comma separate list of the repo names +# 2. (Required) PROTOBUF_RUNTIME_VERSION: Protobuf runtime version to test again +# 3. REPOS_INSTALLED_LOCALLY: Flag (if set to "true) will determine if the repo +# needs to be cloned from github and re-installed locally to the m2. If the artifact +# is already installed locally, this saves time as the artifact is not again compiled +# and built. # Comma-delimited list of repos to test with the local java-shared-dependencies if [ -z "${REPOS_UNDER_TEST}" ]; then From 2c5e5279264ebc7599b88efbbdcff4ec15b4715a Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Fri, 7 Mar 2025 15:26:14 -0500 Subject: [PATCH 24/34] chore: Remove local install repo from source test --- ...ream_protobuf_compatibility_check_nightly.yaml | 2 +- .../downstream-protobuf-source-compatibility.sh | 15 ++++----------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml b/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml index 5959afd1d9..0cdca82d32 100644 --- a/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml +++ b/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml @@ -63,7 +63,7 @@ jobs: - name: Print Protobuf-Java testing version run: echo "Testing with Protobuf-Java v${{ matrix.protobuf-version }}" - name: Perform downstream source compatibility testing - run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version }}" REPOS_INSTALLED_LOCALLY="true" ./.kokoro/nightly/downstream-protobuf-source-compatibility.sh + run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version }}" ./.kokoro/nightly/downstream-protobuf-source-compatibility.sh - name: Perform downstream binary compatibility testing run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version }}" REPOS_INSTALLED_LOCALLY="true" ./.kokoro/nightly/downstream-protobuf-binary-compatibility.sh diff --git a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh index 7e0dca16b5..dadb6d4817 100755 --- a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh @@ -15,13 +15,9 @@ set -eo pipefail -# There are three Env Vars that this script uses. +# There are two Env Vars that this script uses. # 1. (Required) REPOS_UNDER_TEST: Comma separate list of the repo names # 2. (Required) PROTOBUF_RUNTIME_VERSION: Protobuf runtime version to test again -# 3. REPOS_INSTALLED_LOCALLY: Flag (if set to "true) will determine if the repo -# needs to be cloned from github and re-installed locally to the m2. If the artifact -# is already installed locally, this saves time as the artifact is not again compiled -# and built. # Comma-delimited list of repos to test with the local java-shared-dependencies if [ -z "${REPOS_UNDER_TEST}" ]; then @@ -38,16 +34,13 @@ if [ -z "${PROTOBUF_RUNTIME_VERSION}" ]; then fi for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma - if [ "${REPOS_INSTALLED_LOCALLY}" != "true" ]; then - # Perform source-compatibility testing on main (latest changes) - git clone "https://github.com/googleapis/$repo.git" --depth=1 - fi - + # Perform source-compatibility testing on main (latest changes) + git clone "https://github.com/googleapis/$repo.git" --depth=1 pushd "$repo" # Compile the Handwritten Library with the Protobuf-Java version to test source compatibility # Run unit tests to help check for any behavior differences (dependant on coverage) - mvn compile -B -V -ntp \ + mvn test -B -V -ntp \ -Dclirr.skip=true \ -Denforcer.skip=true \ -Dmaven.javadoc.skip=true \ From eaf2389159b179e7605676e08e2c873b57f0ce11 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Fri, 7 Mar 2025 15:36:06 -0500 Subject: [PATCH 25/34] chore: Install artifacts for binary --- ..._protobuf_compatibility_check_nightly.yaml | 12 +-------- ...ownstream-protobuf-binary-compatibility.sh | 27 +++++-------------- ...ownstream-protobuf-source-compatibility.sh | 5 +--- 3 files changed, 8 insertions(+), 36 deletions(-) diff --git a/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml b/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml index 0cdca82d32..df107ac4a8 100644 --- a/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml +++ b/.github/workflows/downstream_protobuf_compatibility_check_nightly.yaml @@ -45,25 +45,15 @@ jobs: steps: - name: Checkout sdk-platform-java repo uses: actions/checkout@v4 - - name: Checkout downstream handwritten repo - uses: actions/checkout@v4 - with: - repository: "googleapis/${{ matrix.repo }}" - path: "${{ matrix.repo }}" - uses: actions/setup-java@v4 with: # Use Java 11 for this as Linkage Checker is only compatible with Java 11 or below java-version: 11 distribution: temurin - - name: Install downstream repo locally - run: | - pushd "${{ matrix.repo }}" - mvn -B -ntp install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip - popd - name: Print Protobuf-Java testing version run: echo "Testing with Protobuf-Java v${{ matrix.protobuf-version }}" - name: Perform downstream source compatibility testing run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version }}" ./.kokoro/nightly/downstream-protobuf-source-compatibility.sh - name: Perform downstream binary compatibility testing - run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version }}" REPOS_INSTALLED_LOCALLY="true" ./.kokoro/nightly/downstream-protobuf-binary-compatibility.sh + run: REPOS_UNDER_TEST="${{ matrix.repo }}" PROTOBUF_RUNTIME_VERSION="${{ matrix.protobuf-version }}" ./.kokoro/nightly/downstream-protobuf-binary-compatibility.sh diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index 834c7a2f24..c5d6b975b2 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -15,14 +15,6 @@ set -eo pipefail -# There are three Env Vars that this script uses. -# 1. (Required) REPOS_UNDER_TEST: Comma separate list of the repo names -# 2. (Required) PROTOBUF_RUNTIME_VERSION: Protobuf runtime version to test again -# 3. REPOS_INSTALLED_LOCALLY: Flag (if set to "true) will determine if the repo -# needs to be cloned from github and re-installed locally to the m2. If the artifact -# is already installed locally, this saves time as the artifact is not again compiled -# and built. - # Comma-delimited list of repos to test with the local java-shared-dependencies if [ -z "${REPOS_UNDER_TEST}" ]; then echo "REPOS_UNDER_TEST must be set to run downstream-protobuf-binary-compatibility.sh" @@ -85,18 +77,11 @@ mvn -B -ntp clean compile -T 1C pushd dependencies for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma - if [ "${REPOS_INSTALLED_LOCALLY}" != "true" ]; then - # Perform testing on main (with latest changes). Shallow copy as history is not important - git clone "https://github.com/googleapis/${repo}.git" --depth=1 - pushd "${repo}" - # Install all repo modules to ~/.m2 (there can be multiple relevant artifacts to test i.e. core, admin, control) - mvn -B -ntp install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip - else - # If installed locally to .m2, then just pull the versions.txt file to parse - mkdir -p "../../${repo}" - pushd "../../${repo}" - curl -Os "https://raw.githubusercontent.com/googleapis/${repo}/refs/heads/main/versions.txt" - fi + # Perform testing on main (with latest changes). Shallow copy as history is not important + git clone "https://github.com/googleapis/${repo}.git" --depth=1 + pushd "${repo}" + # Install all repo modules to ~/.m2 (there can be multiple relevant artifacts to test i.e. core, admin, control) + mvn -B -ntp install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip artifact_list="" if [ "${repo}" == "google-cloud-java" ]; then @@ -114,7 +99,7 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma # The `-s` argument filters the linkage check problems that stem from the artifact program_args="-r --artifacts ${artifact_list},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${artifact_list}" echo "Linkage Checker Program Arguments: ${program_args}" - mvn -B -ntp exec:java -Dexec.mainClass="com.google.cloud.tools.opensource.classpath.LinkageCheckerMain" -Dexec.args="${program_args}" -P exec-linkage-checker + mvn -B -ntp exec:java -Dexec.args="${program_args}" -P exec-linkage-checker fi echo "done" done diff --git a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh index dadb6d4817..669ea2ccc1 100755 --- a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh @@ -15,10 +15,6 @@ set -eo pipefail -# There are two Env Vars that this script uses. -# 1. (Required) REPOS_UNDER_TEST: Comma separate list of the repo names -# 2. (Required) PROTOBUF_RUNTIME_VERSION: Protobuf runtime version to test again - # Comma-delimited list of repos to test with the local java-shared-dependencies if [ -z "${REPOS_UNDER_TEST}" ]; then echo "REPOS_UNDER_TEST must be set to run downstream-protobuf-source-compatibility.sh" @@ -46,4 +42,5 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma -Dmaven.javadoc.skip=true \ -Dprotobuf.version=${PROTOBUF_RUNTIME_VERSION} \ -T 1C + popd done \ No newline at end of file From e11cb7b2d0ee1767ed8700edf37e13b0e17c2058 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Fri, 7 Mar 2025 16:21:53 -0500 Subject: [PATCH 26/34] chore: Revert back to original source changes --- .kokoro/nightly/downstream-protobuf-source-compatibility.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh index 669ea2ccc1..c7caeb724c 100755 --- a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh @@ -36,7 +36,7 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma # Compile the Handwritten Library with the Protobuf-Java version to test source compatibility # Run unit tests to help check for any behavior differences (dependant on coverage) - mvn test -B -V -ntp \ + mvn clean test -B -V -ntp \ -Dclirr.skip=true \ -Denforcer.skip=true \ -Dmaven.javadoc.skip=true \ From ab4bd158c4d5a2fed0ebd1d530ece2434106b0e1 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 10 Mar 2025 12:44:30 -0400 Subject: [PATCH 27/34] chore: Address PR comments --- .kokoro/nightly/common.sh | 31 +++++++++++++++++++ ...ownstream-protobuf-binary-compatibility.sh | 26 +++++++--------- ...ownstream-protobuf-source-compatibility.sh | 15 ++------- 3 files changed, 45 insertions(+), 27 deletions(-) create mode 100644 .kokoro/nightly/common.sh diff --git a/.kokoro/nightly/common.sh b/.kokoro/nightly/common.sh new file mode 100644 index 0000000000..ee30ec3c74 --- /dev/null +++ b/.kokoro/nightly/common.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Checks that the protobuf compatibility scripts provide non-empty input +function validate_protobuf_compatibility_script_inputs { + # Comma-delimited list of repos to test + if [ -z "${REPOS_UNDER_TEST}" ]; then + echo "REPOS_UNDER_TEST Env Var must be set. This script expects a" + echo "comma-delimited list: i.e REPOS_UNDER_TEST=\"java-bigtable,java-bigquery\"" + exit 1 + fi + + # A single version of Protobuf-Java runtime to test + if [ -z "${PROTOBUF_RUNTIME_VERSION}" ]; then + echo "PROTOBUF_RUNTIME_VERSION Env Var must be set. This script expects a single " + echo "Protobuf-Java runtime version i.e. PROTOBUF_RUNTIME_VERSION=\"4.28.3\"" + exit 1 + fi +} \ No newline at end of file diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index c5d6b975b2..20c99ca7fb 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -15,19 +15,10 @@ set -eo pipefail -# Comma-delimited list of repos to test with the local java-shared-dependencies -if [ -z "${REPOS_UNDER_TEST}" ]; then - echo "REPOS_UNDER_TEST must be set to run downstream-protobuf-binary-compatibility.sh" - echo "Expects a comma-delimited list: i.e REPOS_UNDER_TEST=\"java-bigtable,java-bigquery\"" - exit 1 -fi +scriptDir=$(realpath "$(dirname "${BASH_SOURCE[0]}")") +source "${scriptDir}/common.sh" -# Version of Protobuf-Java runtime to compile with -if [ -z "${PROTOBUF_RUNTIME_VERSION}" ]; then - echo "PROTOBUF_RUNTIME_VERSION must be set to run downstream-protobuf-binary-compatibility.sh" - echo "Expects a single Protobuf-Java runtime version i.e. PROTOBUF_RUNTIME_VERSION=\"4.28.3\"" - exit 1 -fi +validate_protobuf_compatibility_script_inputs # Create two mappings of possible API names (Key: Maven Artifact ID Prefix, Value: Maven Group ID) # for the libraries that should be tested. @@ -36,6 +27,7 @@ declare -A monorepo_handwritten_libraries monorepo_handwritten_libraries["grafeas"]="io.grafeas" monorepo_handwritten_libraries["google-cloud-vertexai"]="com.google.cloud" monorepo_handwritten_libraries["google-cloud-resourcemanager"]="com.google.cloud" +monorepo_handwritten_libraries["google-cloud-translate"]="com.google.cloud" # 2. These are the mappings of all the downstream handwritten libraries' artifacts declare -A downstream_handwritten_libraries @@ -48,8 +40,10 @@ function build_artifact_list() { for artifact_id_prefix in "${!api_maven_mapping[@]}"; do group_id="${api_maven_mapping[${artifact_id_prefix}]}" - # Match all artifacts that start with the artifact_id_prefix to exclude any proto and grpc modules. - repo_artifact_list=$(cat "versions.txt" | grep "^${artifact_id_prefix}" || true) + # Match all artifacts that start with the $artifact_id_prefix or grpc to exclude any proto modules. + # grpc-* module are included as gRPC-Java controls their Protobuf version and may generate code + # using a different version of Protobuf + repo_artifact_list=$(cat "versions.txt" | grep -E "^(${artifact_id_prefix}|grpc)" || true) # Only proceed if there are matching elements if [ -n "${repo_artifact_list}" ]; then @@ -100,8 +94,10 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma program_args="-r --artifacts ${artifact_list},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${artifact_list}" echo "Linkage Checker Program Arguments: ${program_args}" mvn -B -ntp exec:java -Dexec.args="${program_args}" -P exec-linkage-checker + else + echo "Unable to find any matching artifacts to test in ${repo}" + exit 1 fi - echo "done" done popd popd diff --git a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh index c7caeb724c..54c121841c 100755 --- a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh @@ -15,19 +15,10 @@ set -eo pipefail -# Comma-delimited list of repos to test with the local java-shared-dependencies -if [ -z "${REPOS_UNDER_TEST}" ]; then - echo "REPOS_UNDER_TEST must be set to run downstream-protobuf-source-compatibility.sh" - echo "Expects a comma-delimited list: i.e REPOS_UNDER_TEST=\"java-bigtable,java-bigquery\"" - exit 1 -fi +scriptDir=$(realpath "$(dirname "${BASH_SOURCE[0]}")") +source "${scriptDir}/common.sh" -# Version of Protobuf-Java runtime to compile with -if [ -z "${PROTOBUF_RUNTIME_VERSION}" ]; then - echo "PROTOBUF_RUNTIME_VERSION must be set to run downstream-protobuf-source-compatibility.sh" - echo "Expects a single Protobuf-Java runtime version i.e. PROTOBUF_RUNTIME_VERSION=\"4.28.3\"" - exit 1 -fi +validate_protobuf_compatibility_script_inputs for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma # Perform source-compatibility testing on main (latest changes) From e58dc768be18c79c0a071a8e13f5d8c536386e16 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 10 Mar 2025 14:13:36 -0400 Subject: [PATCH 28/34] chore: Address PR comments --- ...ownstream-protobuf-binary-compatibility.sh | 36 ++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index 20c99ca7fb..a1fa4f56d1 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -40,10 +40,8 @@ function build_artifact_list() { for artifact_id_prefix in "${!api_maven_mapping[@]}"; do group_id="${api_maven_mapping[${artifact_id_prefix}]}" - # Match all artifacts that start with the $artifact_id_prefix or grpc to exclude any proto modules. - # grpc-* module are included as gRPC-Java controls their Protobuf version and may generate code - # using a different version of Protobuf - repo_artifact_list=$(cat "versions.txt" | grep -E "^(${artifact_id_prefix}|grpc)" || true) + # Match all artifacts that start with the $artifact_id_prefix exclude any non-relevant modules. + repo_artifact_list=$(cat "versions.txt" | grep -E "^${artifact_id_prefix}" || true) # Only proceed if there are matching elements if [ -n "${repo_artifact_list}" ]; then @@ -60,6 +58,26 @@ function build_artifact_list() { artifact_list="${artifact_list},${repo_artifact_list}" fi fi + + # grpc-* module are included as gRPC-Java controls their Protobuf version and may generate code + # using a different version of Protobuf + grpc_repo_artifact_list=$(cat "versions.txt" | grep -E "^grpc" || true) + + # Only proceed if there are matching elements + if [ -n "${grpc_repo_artifact_list}" ]; then + # Exclude any matches to BOM artifacts or emulators. The repo artifact list will look like: + # "com.google.cloud:google-cloud-accessapproval:2.60.0-SNAPSHOT,com.google.cloud:google-cloud-aiplatform:3.60.0-SNAPSHOT," + grpc_repo_artifact_list=$(echo "${grpc_repo_artifact_list}" | awk -F: "{\$1=\"${group_id}:\"\$1; \$2=\"\"; print}" OFS=: | sed 's/::/:/' | tr '\n' ',') + # Remove the trailing comma after the last entry + grpc_repo_artifact_list=${grpc_repo_artifact_list%,} + + # The first entry added is not separated with a comma. Avoids generating `,{ARTIFACT_LIST}` + if [ -z "${artifact_list}" ]; then + grpc_artifact_list="${grpc_repo_artifact_list}" + else + grpc_artifact_list="${grpc_artifact_list},${grpc_repo_artifact_list}" + fi + fi done } @@ -78,6 +96,7 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma mvn -B -ntp install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip artifact_list="" + grpc_artifact_list="" if [ "${repo}" == "google-cloud-java" ]; then build_artifact_list monorepo_handwritten_libraries else @@ -89,11 +108,18 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma echo "Artifact List: ${artifact_list}" # Only run Linkage Checker if the repo has any relevant artifacts to test for - if [ -n "${artifact_list}" ]; then + if [ -n "${artifact_list}" ] && [ -n "${grpc_artifact_list}" ]; then # The `-s` argument filters the linkage check problems that stem from the artifact + # There are two calls to Linkage Checker: 1. repo's handwritten modules 2. repo's gRPC modules + # This is because mvn has a limit on the number of program arguments you can pass in + program_args="-r --artifacts ${artifact_list},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${artifact_list}" echo "Linkage Checker Program Arguments: ${program_args}" mvn -B -ntp exec:java -Dexec.args="${program_args}" -P exec-linkage-checker + + program_args="-r --artifacts ${artifact_list},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${grpc_artifact_list}" + echo "Linkage Checker Program Arguments for gRPC Modules: ${program_args}" + mvn -B -ntp exec:java -Dexec.args="${program_args}" -P exec-linkage-checker else echo "Unable to find any matching artifacts to test in ${repo}" exit 1 From ee4177b09cf937f2dbc88e028a3083fd3d464a71 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 10 Mar 2025 14:14:07 -0400 Subject: [PATCH 29/34] chore: Address PR comments --- .kokoro/nightly/downstream-protobuf-binary-compatibility.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index a1fa4f56d1..66665a47de 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -114,10 +114,12 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma # This is because mvn has a limit on the number of program arguments you can pass in program_args="-r --artifacts ${artifact_list},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${artifact_list}" + echo "Running Linkage Checker on the repo's handwritten modules" echo "Linkage Checker Program Arguments: ${program_args}" mvn -B -ntp exec:java -Dexec.args="${program_args}" -P exec-linkage-checker program_args="-r --artifacts ${artifact_list},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${grpc_artifact_list}" + echo "Running Linkage Checker on the repo's gRPC modules" echo "Linkage Checker Program Arguments for gRPC Modules: ${program_args}" mvn -B -ntp exec:java -Dexec.args="${program_args}" -P exec-linkage-checker else From 3b797416dfcd4e612869506b59ddbe85a9f37b7c Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 10 Mar 2025 14:17:15 -0400 Subject: [PATCH 30/34] chore: Run if there are either handwritten modules or gRPC modules --- .kokoro/nightly/downstream-protobuf-binary-compatibility.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index 66665a47de..ac3e095462 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -108,7 +108,7 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma echo "Artifact List: ${artifact_list}" # Only run Linkage Checker if the repo has any relevant artifacts to test for - if [ -n "${artifact_list}" ] && [ -n "${grpc_artifact_list}" ]; then + if [ -n "${artifact_list}" ] || [ -n "${grpc_artifact_list}" ]; then # The `-s` argument filters the linkage check problems that stem from the artifact # There are two calls to Linkage Checker: 1. repo's handwritten modules 2. repo's gRPC modules # This is because mvn has a limit on the number of program arguments you can pass in From ad8544a9a8aac1f8f7205ae71a32400fb4593c25 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 10 Mar 2025 14:36:37 -0400 Subject: [PATCH 31/34] chore: Fix issue with gRPC modules to test --- ...ownstream-protobuf-binary-compatibility.sh | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index ac3e095462..686b025f3b 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -65,14 +65,12 @@ function build_artifact_list() { # Only proceed if there are matching elements if [ -n "${grpc_repo_artifact_list}" ]; then - # Exclude any matches to BOM artifacts or emulators. The repo artifact list will look like: - # "com.google.cloud:google-cloud-accessapproval:2.60.0-SNAPSHOT,com.google.cloud:google-cloud-aiplatform:3.60.0-SNAPSHOT," - grpc_repo_artifact_list=$(echo "${grpc_repo_artifact_list}" | awk -F: "{\$1=\"${group_id}:\"\$1; \$2=\"\"; print}" OFS=: | sed 's/::/:/' | tr '\n' ',') + grpc_repo_artifact_list=$(echo "${grpc_repo_artifact_list}" | awk -F: "{\$1=\"com.google.api.grpc:\"\$1; \$2=\"\"; print}" OFS=: | sed 's/::/:/' | tr '\n' ',') # Remove the trailing comma after the last entry grpc_repo_artifact_list=${grpc_repo_artifact_list%,} # The first entry added is not separated with a comma. Avoids generating `,{ARTIFACT_LIST}` - if [ -z "${artifact_list}" ]; then + if [ -z "${grpc_artifact_list}" ]; then grpc_artifact_list="${grpc_repo_artifact_list}" else grpc_artifact_list="${grpc_artifact_list},${grpc_repo_artifact_list}" @@ -107,24 +105,26 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma popd echo "Artifact List: ${artifact_list}" - # Only run Linkage Checker if the repo has any relevant artifacts to test for - if [ -n "${artifact_list}" ] || [ -n "${grpc_artifact_list}" ]; then - # The `-s` argument filters the linkage check problems that stem from the artifact - # There are two calls to Linkage Checker: 1. repo's handwritten modules 2. repo's gRPC modules - # This is because mvn has a limit on the number of program arguments you can pass in - + # The `-s` argument filters the linkage check problems that stem from the artifact + # There are two calls to Linkage Checker: 1. repo's handwritten modules 2. repo's gRPC modules + # This is because mvn has a limit on the number of program arguments you can pass in + if [ -n "${artifact_list}" ]; then program_args="-r --artifacts ${artifact_list},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${artifact_list}" echo "Running Linkage Checker on the repo's handwritten modules" echo "Linkage Checker Program Arguments: ${program_args}" mvn -B -ntp exec:java -Dexec.args="${program_args}" -P exec-linkage-checker + else + echo "Unable to find any matching artifacts to test in ${repo}" + exit 1 + fi - program_args="-r --artifacts ${artifact_list},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${grpc_artifact_list}" + echo "gRPC Artifact List: ${grpc_artifact_list}" + # Some downstream handwritten artifacts do not have gRPC support, do not fail if there are no artifacts + if [ -n "${grpc_artifact_list}" ]; then + program_args="-r --artifacts ${grpc_artifact_list},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${grpc_artifact_list}" echo "Running Linkage Checker on the repo's gRPC modules" echo "Linkage Checker Program Arguments for gRPC Modules: ${program_args}" mvn -B -ntp exec:java -Dexec.args="${program_args}" -P exec-linkage-checker - else - echo "Unable to find any matching artifacts to test in ${repo}" - exit 1 fi done popd From 6b4651088d2e75854cec162acae4ce5bbaa2ab36 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 10 Mar 2025 15:24:46 -0400 Subject: [PATCH 32/34] chore: Add a few grpc modules for google-cloud-java --- ...ownstream-protobuf-binary-compatibility.sh | 37 ++++--------------- 1 file changed, 7 insertions(+), 30 deletions(-) diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index 686b025f3b..df4e77322c 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -28,10 +28,17 @@ monorepo_handwritten_libraries["grafeas"]="io.grafeas" monorepo_handwritten_libraries["google-cloud-vertexai"]="com.google.cloud" monorepo_handwritten_libraries["google-cloud-resourcemanager"]="com.google.cloud" monorepo_handwritten_libraries["google-cloud-translate"]="com.google.cloud" +# Test a few grpc-* modules as gRPC-Java controls their Protobuf version and may generate code using +# a different version of Protobuf. Not all grpc-* modules are tested as this may build a massive list +# of artifacts. Maven has a limit on the number of arguments it can take (google-cloud-java surpasses that) +monorepo_handwritten_libraries["grpc-google-cloud-vertexai"]="com.google.api.grpc" +monorepo_handwritten_libraries["grpc-google-cloud-resourcemanager"]="com.google.api.grpc" +monorepo_handwritten_libraries["grpc-google-cloud-translate"]="com.google.api.grpc" # 2. These are the mappings of all the downstream handwritten libraries' artifacts declare -A downstream_handwritten_libraries downstream_handwritten_libraries["google-cloud"]="com.google.cloud" +downstream_handwritten_libraries["grpc-google-cloud"]="com.google.api.grpc" # Builds a string output to `artifact_list`. It contains a comma separate list of Maven GAV coordinates. Parses # the `versions.txt` file by searching for the matching artifact_id_prefix to get the corresponding version. @@ -58,24 +65,6 @@ function build_artifact_list() { artifact_list="${artifact_list},${repo_artifact_list}" fi fi - - # grpc-* module are included as gRPC-Java controls their Protobuf version and may generate code - # using a different version of Protobuf - grpc_repo_artifact_list=$(cat "versions.txt" | grep -E "^grpc" || true) - - # Only proceed if there are matching elements - if [ -n "${grpc_repo_artifact_list}" ]; then - grpc_repo_artifact_list=$(echo "${grpc_repo_artifact_list}" | awk -F: "{\$1=\"com.google.api.grpc:\"\$1; \$2=\"\"; print}" OFS=: | sed 's/::/:/' | tr '\n' ',') - # Remove the trailing comma after the last entry - grpc_repo_artifact_list=${grpc_repo_artifact_list%,} - - # The first entry added is not separated with a comma. Avoids generating `,{ARTIFACT_LIST}` - if [ -z "${grpc_artifact_list}" ]; then - grpc_artifact_list="${grpc_repo_artifact_list}" - else - grpc_artifact_list="${grpc_artifact_list},${grpc_repo_artifact_list}" - fi - fi done } @@ -94,7 +83,6 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma mvn -B -ntp install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip artifact_list="" - grpc_artifact_list="" if [ "${repo}" == "google-cloud-java" ]; then build_artifact_list monorepo_handwritten_libraries else @@ -106,8 +94,6 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma echo "Artifact List: ${artifact_list}" # The `-s` argument filters the linkage check problems that stem from the artifact - # There are two calls to Linkage Checker: 1. repo's handwritten modules 2. repo's gRPC modules - # This is because mvn has a limit on the number of program arguments you can pass in if [ -n "${artifact_list}" ]; then program_args="-r --artifacts ${artifact_list},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${artifact_list}" echo "Running Linkage Checker on the repo's handwritten modules" @@ -117,15 +103,6 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma echo "Unable to find any matching artifacts to test in ${repo}" exit 1 fi - - echo "gRPC Artifact List: ${grpc_artifact_list}" - # Some downstream handwritten artifacts do not have gRPC support, do not fail if there are no artifacts - if [ -n "${grpc_artifact_list}" ]; then - program_args="-r --artifacts ${grpc_artifact_list},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${grpc_artifact_list}" - echo "Running Linkage Checker on the repo's gRPC modules" - echo "Linkage Checker Program Arguments for gRPC Modules: ${program_args}" - mvn -B -ntp exec:java -Dexec.args="${program_args}" -P exec-linkage-checker - fi done popd popd From 2fb338f4649c49929dee3abb838281c4d4d3e0cf Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Wed, 12 Mar 2025 18:24:44 -0400 Subject: [PATCH 33/34] chore: Address PR comments --- ...ownstream-protobuf-binary-compatibility.sh | 100 ++++++++---------- ...ownstream-protobuf-source-compatibility.sh | 3 + 2 files changed, 47 insertions(+), 56 deletions(-) diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index df4e77322c..4a315dc481 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -20,50 +20,44 @@ source "${scriptDir}/common.sh" validate_protobuf_compatibility_script_inputs -# Create two mappings of possible API names (Key: Maven Artifact ID Prefix, Value: Maven Group ID) -# for the libraries that should be tested. -# 1. These are special handwritten libraries in google-cloud-java that should be tested -declare -A monorepo_handwritten_libraries -monorepo_handwritten_libraries["grafeas"]="io.grafeas" -monorepo_handwritten_libraries["google-cloud-vertexai"]="com.google.cloud" -monorepo_handwritten_libraries["google-cloud-resourcemanager"]="com.google.cloud" -monorepo_handwritten_libraries["google-cloud-translate"]="com.google.cloud" -# Test a few grpc-* modules as gRPC-Java controls their Protobuf version and may generate code using -# a different version of Protobuf. Not all grpc-* modules are tested as this may build a massive list -# of artifacts. Maven has a limit on the number of arguments it can take (google-cloud-java surpasses that) -monorepo_handwritten_libraries["grpc-google-cloud-vertexai"]="com.google.api.grpc" -monorepo_handwritten_libraries["grpc-google-cloud-resourcemanager"]="com.google.api.grpc" -monorepo_handwritten_libraries["grpc-google-cloud-translate"]="com.google.api.grpc" +# Declare a map of downstream handwritten libraries and the relevant artifacts to test. The map stores a +# K/V pairing of (Key: repo name, Value: comma separate list of Group ID:Artifact ID pairings). Note: The +# value list doesn't hold the version and this needs to be parsed from the repo's versions.txt file +declare -A repo_linkage_checker_arguments +repo_linkage_checker_arguments["google-cloud-java"]="io.grafeas:grafeas,com.google.cloud:google-cloud-vertexai,com.google.cloud:google-cloud-resourcemanager,com.google.cloud:google-cloud-translate,com.google.api.grpc:grpc-google-cloud-vertexai,com.google.api.grpc:grpc-google-cloud-resourcemanager,com.google.api.grpc:grpc-google-cloud-translate" +repo_linkage_checker_arguments["java-bigtable"]="com.google.cloud:google-cloud-bigtable,com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2,com.google.api.grpc:grpc-google-cloud-bigtable-v2" +repo_linkage_checker_arguments["java-bigquery"]="com.google.cloud:google-cloud-bigquery" +repo_linkage_checker_arguments["java-bigquerystorage"]="com.google.cloud:google-cloud-bigquerystorage,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1alpha" +repo_linkage_checker_arguments["java-datastore"]="com.google.cloud:google-cloud-datastore,com.google.cloud.datastre:datastore-v1-proto-client,com.google.api.grpc:grpc-google-cloud-datastore-admin-v1" +repo_linkage_checker_arguments["java-firestore"]="com.google.cloud:google-cloud-firestore,com.google.cloud:google-cloud-firestore-admin,com.google.api.grpc:grpc-google-cloud-firestore-admin-v1,com.google.api.grpc:grpc-google-cloud-firestore-v1" +repo_linkage_checker_arguments["java-logging"]="com.google.cloud:google-cloud-logging,com.google.api.grpc:grpc-google-cloud-logging-v2" +repo_linkage_checker_arguments["java-logging-logback"]="com.google.cloud:google-cloud-logging-logback" +repo_linkage_checker_arguments["java-pubsub"]="com.google.cloud:google-cloud-pubsub,com.google.api.grpc:grpc-google-cloud-pubsub-v1" +repo_linkage_checker_arguments["java-pubsublite"]="com.google.cloud:google-cloud-pubsublite,com.google.api.grpc:grpc-google-cloud-pubsublite-v1" +repo_linkage_checker_arguments["java-spanner-jdbc"]="com.google.cloud:google-cloud-spanner-jdbc" +repo_linkage_checker_arguments["java-spanner"]="com.google.cloud:google-cloud-spanner,com.google.cloud:google-cloud-spanner-executor,com.google.api.grpc:grpc-google-cloud-spanner-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1,com.google.api.grpc:grpc-google-cloud-spanner-executor-v1" +repo_linkage_checker_arguments["java-storage"]="com.google.cloud:google-cloud-storage,com.google.api.grpc:gapic-google-cloud-storage-v2,com.google.api.grpc:grpc-google-cloud-storage-v2,com.google.cloud:google-cloud-storage-control,com.google.api.grpc:grpc-google-cloud-storage-control-v2" +repo_linkage_checker_arguments["java-storage-nio"]="com.google.cloud:google-cloud-nio" -# 2. These are the mappings of all the downstream handwritten libraries' artifacts -declare -A downstream_handwritten_libraries -downstream_handwritten_libraries["google-cloud"]="com.google.cloud" -downstream_handwritten_libraries["grpc-google-cloud"]="com.google.api.grpc" +# This function requires access to the versions.txt to retrieve the versions for the artifacts +# It will try to match the artifact_id in the versions.txt file and attach it to form the GAV +# The GAV list is required by Linkage Checker as program arguments +function build_program_arguments() { + artifact_list="${repo_linkage_checker_arguments[$1]}" -# Builds a string output to `artifact_list`. It contains a comma separate list of Maven GAV coordinates. Parses -# the `versions.txt` file by searching for the matching artifact_id_prefix to get the corresponding version. -function build_artifact_list() { - local -n api_maven_mapping=$1 - for artifact_id_prefix in "${!api_maven_mapping[@]}"; do - group_id="${api_maven_mapping[${artifact_id_prefix}]}" + for artifact in ${artifact_list//,/ }; do # Split on comma + artifact_id=$(echo "${artifact}" | cut -d ':' -f2) - # Match all artifacts that start with the $artifact_id_prefix exclude any non-relevant modules. - repo_artifact_list=$(cat "versions.txt" | grep -E "^${artifact_id_prefix}" || true) + # The grep query tries to match `{artifact_id}:{released_version}:{current_version}`. + # The artifact_id must be exact otherwise multiple entries may match + version=$(cat "versions.txt" | grep -E "^${artifact_id}:.*:.*$" | cut -d ':' -f3) + repo_gav_coordinate="${artifact}:${version}" - # Only proceed if there are matching elements - if [ -n "${repo_artifact_list}" ]; then - # Exclude any matches to BOM artifacts or emulators. The repo artifact list will look like: - # "com.google.cloud:google-cloud-accessapproval:2.60.0-SNAPSHOT,com.google.cloud:google-cloud-aiplatform:3.60.0-SNAPSHOT," - repo_artifact_list=$(echo "${repo_artifact_list}" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: "{\$1=\"${group_id}:\"\$1; \$2=\"\"; print}" OFS=: | sed 's/::/:/' | tr '\n' ',') - # Remove the trailing comma after the last entry - repo_artifact_list=${repo_artifact_list%,} - - # The first entry added is not separated with a comma. Avoids generating `,{ARTIFACT_LIST}` - if [ -z "${artifact_list}" ]; then - artifact_list="${repo_artifact_list}" - else - artifact_list="${artifact_list},${repo_artifact_list}" - fi + # The first entry added is not separated with a comma. Avoids generating `,{ARTIFACT_LIST}` + if [ -z "${linkage_checker_arguments}" ]; then + linkage_checker_arguments="${repo_gav_coordinate}" + else + linkage_checker_arguments="${linkage_checker_arguments},${repo_gav_coordinate}" fi done } @@ -75,6 +69,9 @@ mvn -B -ntp clean compile -T 1C # Linkage Checker tool resides in the /dependencies subfolder pushd dependencies +# REPOS_UNDER_TEST Env Var accepts a comma separated list of googleapis repos to test. For Github CI, +# this will be a single repo as Github will build a matrix of repos with each repo being tested in parallel. +# For local invocation, you can pass a list of repos to test multiple repos together. for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma # Perform testing on main (with latest changes). Shallow copy as history is not important git clone "https://github.com/googleapis/${repo}.git" --depth=1 @@ -82,27 +79,18 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma # Install all repo modules to ~/.m2 (there can be multiple relevant artifacts to test i.e. core, admin, control) mvn -B -ntp install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip - artifact_list="" - if [ "${repo}" == "google-cloud-java" ]; then - build_artifact_list monorepo_handwritten_libraries - else - build_artifact_list downstream_handwritten_libraries - fi + linkage_checker_arguments="" + build_program_arguments "${repo}" # Linkage Checker /dependencies popd - echo "Artifact List: ${artifact_list}" + echo "Artifact List: ${linkage_checker_arguments}" # The `-s` argument filters the linkage check problems that stem from the artifact - if [ -n "${artifact_list}" ]; then - program_args="-r --artifacts ${artifact_list},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${artifact_list}" - echo "Running Linkage Checker on the repo's handwritten modules" - echo "Linkage Checker Program Arguments: ${program_args}" - mvn -B -ntp exec:java -Dexec.args="${program_args}" -P exec-linkage-checker - else - echo "Unable to find any matching artifacts to test in ${repo}" - exit 1 - fi + program_args="-r --artifacts ${linkage_checker_arguments},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${linkage_checker_arguments}" + echo "Running Linkage Checker on the repo's handwritten modules" + echo "Linkage Checker Program Arguments: ${program_args}" + mvn -B -ntp exec:java -Dexec.args="${program_args}" -P exec-linkage-checker done popd popd diff --git a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh index 54c121841c..b718491689 100755 --- a/.kokoro/nightly/downstream-protobuf-source-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-source-compatibility.sh @@ -20,6 +20,9 @@ source "${scriptDir}/common.sh" validate_protobuf_compatibility_script_inputs +# REPOS_UNDER_TEST Env Var accepts a comma separated list of googleapis repos to test. For Github CI, +# this will be a single repo as Github will build a matrix of repos with each repo being tested in parallel. +# For local invocation, you can pass a list of repos to test multiple repos together. for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma # Perform source-compatibility testing on main (latest changes) git clone "https://github.com/googleapis/$repo.git" --depth=1 From 806fece87bf4e931d870e84253938acc8083c966 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Wed, 12 Mar 2025 22:48:06 -0400 Subject: [PATCH 34/34] chore: update google-cloud-java deps --- .kokoro/nightly/downstream-protobuf-binary-compatibility.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index 4a315dc481..46dd4e967c 100755 --- a/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -24,7 +24,7 @@ validate_protobuf_compatibility_script_inputs # K/V pairing of (Key: repo name, Value: comma separate list of Group ID:Artifact ID pairings). Note: The # value list doesn't hold the version and this needs to be parsed from the repo's versions.txt file declare -A repo_linkage_checker_arguments -repo_linkage_checker_arguments["google-cloud-java"]="io.grafeas:grafeas,com.google.cloud:google-cloud-vertexai,com.google.cloud:google-cloud-resourcemanager,com.google.cloud:google-cloud-translate,com.google.api.grpc:grpc-google-cloud-vertexai,com.google.api.grpc:grpc-google-cloud-resourcemanager,com.google.api.grpc:grpc-google-cloud-translate" +repo_linkage_checker_arguments["google-cloud-java"]="io.grafeas:grafeas,com.google.cloud:google-cloud-vertexai,com.google.cloud:google-cloud-resourcemanager,com.google.cloud:google-cloud-translate,com.google.api.grpc:grpc-google-cloud-vertexai-v1,com.google.api.grpc:grpc-google-cloud-vertexai-v1beta1,com.google.api.grpc:grpc-google-cloud-resourcemanager-v3,com.google.api.grpc:grpc-google-cloud-translate-v3,com.google.api.grpc:grpc-google-cloud-translate-v3beta1" repo_linkage_checker_arguments["java-bigtable"]="com.google.cloud:google-cloud-bigtable,com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2,com.google.api.grpc:grpc-google-cloud-bigtable-v2" repo_linkage_checker_arguments["java-bigquery"]="com.google.cloud:google-cloud-bigquery" repo_linkage_checker_arguments["java-bigquerystorage"]="com.google.cloud:google-cloud-bigquerystorage,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1alpha"