Skip to content

Commit 2fb338f

Browse files
lqiu96lqiu96
committed
chore: Address PR comments
1 parent 6b46510 commit 2fb338f

File tree

2 files changed

+47
-56
lines changed

2 files changed

+47
-56
lines changed

‎.kokoro/nightly/downstream-protobuf-binary-compatibility.sh‎

Lines changed: 44 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -20,50 +20,44 @@ source "${scriptDir}/common.sh"
2020

2121
validate_protobuf_compatibility_script_inputs
2222

23-
# Create two mappings of possible API names (Key: Maven Artifact ID Prefix, Value: Maven Group ID)
24-
# for the libraries that should be tested.
25-
# 1. These are special handwritten libraries in google-cloud-java that should be tested
26-
declare -A monorepo_handwritten_libraries
27-
monorepo_handwritten_libraries["grafeas"]="io.grafeas"
28-
monorepo_handwritten_libraries["google-cloud-vertexai"]="com.google.cloud"
29-
monorepo_handwritten_libraries["google-cloud-resourcemanager"]="com.google.cloud"
30-
monorepo_handwritten_libraries["google-cloud-translate"]="com.google.cloud"
31-
# Test a few grpc-* modules as gRPC-Java controls their Protobuf version and may generate code using
32-
# a different version of Protobuf. Not all grpc-* modules are tested as this may build a massive list
33-
# of artifacts. Maven has a limit on the number of arguments it can take (google-cloud-java surpasses that)
34-
monorepo_handwritten_libraries["grpc-google-cloud-vertexai"]="com.google.api.grpc"
35-
monorepo_handwritten_libraries["grpc-google-cloud-resourcemanager"]="com.google.api.grpc"
36-
monorepo_handwritten_libraries["grpc-google-cloud-translate"]="com.google.api.grpc"
23+
# Declare a map of downstream handwritten libraries and the relevant artifacts to test. The map stores a
24+
# K/V pairing of (Key: repo name, Value: comma separate list of Group ID:Artifact ID pairings). Note: The
25+
# value list doesn't hold the version and this needs to be parsed from the repo's versions.txt file
26+
declare -A repo_linkage_checker_arguments
27+
repo_linkage_checker_arguments["google-cloud-java"]="io.grafeas:grafeas,com.google.cloud:google-cloud-vertexai,com.google.cloud:google-cloud-resourcemanager,com.google.cloud:google-cloud-translate,com.google.api.grpc:grpc-google-cloud-vertexai,com.google.api.grpc:grpc-google-cloud-resourcemanager,com.google.api.grpc:grpc-google-cloud-translate"
28+
repo_linkage_checker_arguments["java-bigtable"]="com.google.cloud:google-cloud-bigtable,com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2,com.google.api.grpc:grpc-google-cloud-bigtable-v2"
29+
repo_linkage_checker_arguments["java-bigquery"]="com.google.cloud:google-cloud-bigquery"
30+
repo_linkage_checker_arguments["java-bigquerystorage"]="com.google.cloud:google-cloud-bigquerystorage,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1alpha"
31+
repo_linkage_checker_arguments["java-datastore"]="com.google.cloud:google-cloud-datastore,com.google.cloud.datastre:datastore-v1-proto-client,com.google.api.grpc:grpc-google-cloud-datastore-admin-v1"
32+
repo_linkage_checker_arguments["java-firestore"]="com.google.cloud:google-cloud-firestore,com.google.cloud:google-cloud-firestore-admin,com.google.api.grpc:grpc-google-cloud-firestore-admin-v1,com.google.api.grpc:grpc-google-cloud-firestore-v1"
33+
repo_linkage_checker_arguments["java-logging"]="com.google.cloud:google-cloud-logging,com.google.api.grpc:grpc-google-cloud-logging-v2"
34+
repo_linkage_checker_arguments["java-logging-logback"]="com.google.cloud:google-cloud-logging-logback"
35+
repo_linkage_checker_arguments["java-pubsub"]="com.google.cloud:google-cloud-pubsub,com.google.api.grpc:grpc-google-cloud-pubsub-v1"
36+
repo_linkage_checker_arguments["java-pubsublite"]="com.google.cloud:google-cloud-pubsublite,com.google.api.grpc:grpc-google-cloud-pubsublite-v1"
37+
repo_linkage_checker_arguments["java-spanner-jdbc"]="com.google.cloud:google-cloud-spanner-jdbc"
38+
repo_linkage_checker_arguments["java-spanner"]="com.google.cloud:google-cloud-spanner,com.google.cloud:google-cloud-spanner-executor,com.google.api.grpc:grpc-google-cloud-spanner-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1,com.google.api.grpc:grpc-google-cloud-spanner-executor-v1"
39+
repo_linkage_checker_arguments["java-storage"]="com.google.cloud:google-cloud-storage,com.google.api.grpc:gapic-google-cloud-storage-v2,com.google.api.grpc:grpc-google-cloud-storage-v2,com.google.cloud:google-cloud-storage-control,com.google.api.grpc:grpc-google-cloud-storage-control-v2"
40+
repo_linkage_checker_arguments["java-storage-nio"]="com.google.cloud:google-cloud-nio"
3741

38-
# 2. These are the mappings of all the downstream handwritten libraries' artifacts
39-
declare -A downstream_handwritten_libraries
40-
downstream_handwritten_libraries["google-cloud"]="com.google.cloud"
41-
downstream_handwritten_libraries["grpc-google-cloud"]="com.google.api.grpc"
42+
# This function requires access to the versions.txt to retrieve the versions for the artifacts
43+
# It will try to match the artifact_id in the versions.txt file and attach it to form the GAV
44+
# The GAV list is required by Linkage Checker as program arguments
45+
function build_program_arguments() {
46+
artifact_list="${repo_linkage_checker_arguments[$1]}"
4247

43-
# Builds a string output to `artifact_list`. It contains a comma separate list of Maven GAV coordinates. Parses
44-
# the `versions.txt` file by searching for the matching artifact_id_prefix to get the corresponding version.
45-
function build_artifact_list() {
46-
local -n api_maven_mapping=$1
47-
for artifact_id_prefix in "${!api_maven_mapping[@]}"; do
48-
group_id="${api_maven_mapping[${artifact_id_prefix}]}"
48+
for artifact in ${artifact_list//,/ }; do # Split on comma
49+
artifact_id=$(echo "${artifact}" | cut -d ':' -f2)
4950

50-
# Match all artifacts that start with the $artifact_id_prefix exclude any non-relevant modules.
51-
repo_artifact_list=$(cat "versions.txt" | grep -E "^${artifact_id_prefix}" || true)
51+
# The grep query tries to match `{artifact_id}:{released_version}:{current_version}`.
52+
# The artifact_id must be exact otherwise multiple entries may match
53+
version=$(cat "versions.txt" | grep -E "^${artifact_id}:.*:.*$" | cut -d ':' -f3)
54+
repo_gav_coordinate="${artifact}:${version}"
5255

53-
# Only proceed if there are matching elements
54-
if [ -n "${repo_artifact_list}" ]; then
55-
# Exclude any matches to BOM artifacts or emulators. The repo artifact list will look like:
56-
# "com.google.cloud:google-cloud-accessapproval:2.60.0-SNAPSHOT,com.google.cloud:google-cloud-aiplatform:3.60.0-SNAPSHOT,"
57-
repo_artifact_list=$(echo "${repo_artifact_list}" | grep -vE "(bom|emulator|google-cloud-java)" | awk -F: "{\$1=\"${group_id}:\"\$1; \$2=\"\"; print}" OFS=: | sed 's/::/:/' | tr '\n' ',')
58-
# Remove the trailing comma after the last entry
59-
repo_artifact_list=${repo_artifact_list%,}
60-
61-
# The first entry added is not separated with a comma. Avoids generating `,{ARTIFACT_LIST}`
62-
if [ -z "${artifact_list}" ]; then
63-
artifact_list="${repo_artifact_list}"
64-
else
65-
artifact_list="${artifact_list},${repo_artifact_list}"
66-
fi
56+
# The first entry added is not separated with a comma. Avoids generating `,{ARTIFACT_LIST}`
57+
if [ -z "${linkage_checker_arguments}" ]; then
58+
linkage_checker_arguments="${repo_gav_coordinate}"
59+
else
60+
linkage_checker_arguments="${linkage_checker_arguments},${repo_gav_coordinate}"
6761
fi
6862
done
6963
}
@@ -75,34 +69,28 @@ mvn -B -ntp clean compile -T 1C
7569
# Linkage Checker tool resides in the /dependencies subfolder
7670
pushd dependencies
7771

72+
# REPOS_UNDER_TEST Env Var accepts a comma separated list of googleapis repos to test. For Github CI,
73+
# this will be a single repo as Github will build a matrix of repos with each repo being tested in parallel.
74+
# For local invocation, you can pass a list of repos to test multiple repos together.
7875
for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma
7976
# Perform testing on main (with latest changes). Shallow copy as history is not important
8077
git clone "https://github.com/googleapis/${repo}.git" --depth=1
8178
pushd "${repo}"
8279
# Install all repo modules to ~/.m2 (there can be multiple relevant artifacts to test i.e. core, admin, control)
8380
mvn -B -ntp install -T 1C -DskipTests -Dclirr.skip -Denforcer.skip
8481

85-
artifact_list=""
86-
if [ "${repo}" == "google-cloud-java" ]; then
87-
build_artifact_list monorepo_handwritten_libraries
88-
else
89-
build_artifact_list downstream_handwritten_libraries
90-
fi
82+
linkage_checker_arguments=""
83+
build_program_arguments "${repo}"
9184

9285
# Linkage Checker /dependencies
9386
popd
9487

95-
echo "Artifact List: ${artifact_list}"
88+
echo "Artifact List: ${linkage_checker_arguments}"
9689
# The `-s` argument filters the linkage check problems that stem from the artifact
97-
if [ -n "${artifact_list}" ]; then
98-
program_args="-r --artifacts ${artifact_list},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${artifact_list}"
99-
echo "Running Linkage Checker on the repo's handwritten modules"
100-
echo "Linkage Checker Program Arguments: ${program_args}"
101-
mvn -B -ntp exec:java -Dexec.args="${program_args}" -P exec-linkage-checker
102-
else
103-
echo "Unable to find any matching artifacts to test in ${repo}"
104-
exit 1
105-
fi
90+
program_args="-r --artifacts ${linkage_checker_arguments},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${linkage_checker_arguments}"
91+
echo "Running Linkage Checker on the repo's handwritten modules"
92+
echo "Linkage Checker Program Arguments: ${program_args}"
93+
mvn -B -ntp exec:java -Dexec.args="${program_args}" -P exec-linkage-checker
10694
done
10795
popd
10896
popd

‎.kokoro/nightly/downstream-protobuf-source-compatibility.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ source "${scriptDir}/common.sh"
2020

2121
validate_protobuf_compatibility_script_inputs
2222

23+
# REPOS_UNDER_TEST Env Var accepts a comma separated list of googleapis repos to test. For Github CI,
24+
# this will be a single repo as Github will build a matrix of repos with each repo being tested in parallel.
25+
# For local invocation, you can pass a list of repos to test multiple repos together.
2326
for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma
2427
# Perform source-compatibility testing on main (latest changes)
2528
git clone "https://github.com/googleapis/$repo.git" --depth=1

0 commit comments

Comments
 (0)