Migrate gsutil usage to gcloud storage (#976)

Author: bhandarivijay-png

Makefile

@@ -132,7 +132,7 @@ upload-e2e-client-binaries: build-grpc-echo build-grpc-interop
 	@cp -n bin/stress_test /tmp/apiproxy-test-presubmit-binaries/stress_test
 	@cp -n bin/grpc_echo_client /tmp/apiproxy-test-presubmit-binaries/grpc_echo_client
 	@cp -n tests/endpoints/grpc_echo/proto/api_descriptor.pb /tmp/apiproxy-test-presubmit-binaries/api_descriptor.pb
-	@gsutil cp -Z -r /tmp/apiproxy-test-presubmit-binaries/* "gs://apiproxy-testing-presubmit-binaries/"
+	@gcloud storage cp --gzip-local-all --recursive /tmp/apiproxy-test-presubmit-binaries/* "gs://apiproxy-testing-presubmit-binaries/"
 
 
 #-----------------------------------------------------------------------------

prow/gcpproxy-coverage.sh

@@ -73,10 +73,10 @@ case "${JOB_TYPE}" in
 esac
 
 # Upload folder.
-gsutil -m rsync -r -d "${ROOT}/generated" "gs://esp-v2-coverage/${PUBLIC_DIRECTORY}"
+gcloud storage rsync --recursive --delete-unmatched-destination-objects "${ROOT}/generated" "gs://esp-v2-coverage/${PUBLIC_DIRECTORY}"
 
 # No browser cache since some directories change often and that would be misleading.
-gsutil -m setmeta -h "Cache-Control:private, max-age=0, no-transform" "gs://esp-v2-coverage/${PUBLIC_DIRECTORY}/**"
+gcloud storage objects update --cache-control="private, max-age=0, no-transform" "gs://esp-v2-coverage/${PUBLIC_DIRECTORY}/**"
 
 echo '======================================================='
 echo '==================== View Coverage ===================='

scripts/all-utilities.sh

@@ -131,7 +131,7 @@ function retry() {
 # Download api Keys from Cloud storage and source the file.
 function set_api_keys() {
   local api_key_directory="$(mktemp -d)"
-  $GSUTIL cp gs://apiproxy-testing-client-secret-files/api_keys  \
+  gcloud storage cp gs://apiproxy-testing-client-secret-files/api_keys  \
     "${api_key_directory}/api_keys"  \
     || error_exit "Failed to download API key file."
 
@@ -142,7 +142,7 @@ function set_api_keys() {
 function get_test_client_key() {
   local remote_file_name=$1
   local key_path=$2
-  $GSUTIL  cp "gs://apiproxy-testing-client-secret-files/$remote_file_name" "$key_path"
+  gcloud storage cp "gs://apiproxy-testing-client-secret-files/$remote_file_name" "$key_path"
   echo -n "$key_path"
   return 0
 }
@@ -279,7 +279,7 @@ function update_tool() {
 
   [[ -z "${TOOLS_BUCKET}" ]] && return 1
   echo "Uploading ${local_path} to ${remote_path}."
-  ${GSUTIL} cp "${local_path}" "${remote_path}"  \
+  gcloud storage cp "${local_path}" "${remote_path}"  \
     || { echo "Failed to upload ${tool_name} to ${TOOLS_BUCKET}";
   return 1; }
   return 0
@@ -293,7 +293,7 @@ function get_tool() {
 
   [[ -z "${TOOLS_BUCKET}" ]] && return 1
   echo "Downloading ${remote_path} to ${local_path}."
-  ${GSUTIL} cp "${remote_path}" "${local_path}"  \
+  gcloud storage cp "${remote_path}" "${local_path}"  \
     || { echo "Failed to upload ${tool_name} to ${TOOLS_BUCKET}";
   return 1; }
   return 0

scripts/release/release-qualify.sh

@@ -114,7 +114,7 @@ function count_stress_failures() {
   mkdir -p "${LOGS}/${SHA}"
 
   echo "Downloading prow logs to '${LOGS}' directory."
-  ${GSUTIL} -m -q cp -r "gs://apiproxy-continuous-long-run/${SHA}/logs/*" "${LOGS}/${SHA}/" 2>&1  \
+  gcloud -q storage cp --recursive "gs://apiproxy-continuous-long-run/${SHA}/logs/*" "${LOGS}/${SHA}/" 2>&1  \
     || error_exit "Failed to download logs from gs://apiproxy-continuous-long-run/${SHA}/logs/*"
 
   python3 "${ROOT}/scripts/release/validate_release.py"  \

src/go/gcsrunner/main/runner.go

@@ -21,7 +21,7 @@
 // Fetches from the bucket at `BUCKET` at the path `CONFIG_FILE_NAME`,
 // Behavior is similar to running:
 //
-// `gsutil cp "gs://${BUCKET}/${CONFIG_FILE_NAME}" envoy.json`
+// `gcloud storage cp "gs://${BUCKET}/${CONFIG_FILE_NAME}" envoy.json`
 //
 // without needing `gsutil` in the image.
 //

tests/e2e/scripts/linux-test-kb-long-run.sh

@@ -56,7 +56,7 @@ fi
 # Download api Keys with restrictions from Cloud storage.
 TEMP_DIR="$(mktemp -d)"
 API_RESTRICTION_KEYS_FILE="${TEMP_DIR}/apiproxy-e2e-key-restriction.json"
-gsutil cp gs://apiproxy-testing-client-secret-files/restricted_api_keys.json  \
+gcloud storage cp gs://apiproxy-testing-client-secret-files/restricted_api_keys.json  \
   "${API_RESTRICTION_KEYS_FILE}"  \
   || error_exit "Failed to download API key with restrictions file."
 

tests/e2e/scripts/prow-utilities.sh

@@ -257,7 +257,8 @@ function upload_logs() {
   local log_dir="${2}"
 
   echo "Uploading content of ${log_dir} to ${remote_dir}"
-  retry -n 3 ${GSUTIL} -h 'Content-Type:text/plain' -m cp -r  \
+  # The gsutil top-level flag `-h` is not supported in `gcloud storage`.
+  retry -n 3 gcloud storage cp --recursive  \
     "${log_dir}" "${remote_dir}"  \
     || echo "Failed to upload ${log_dir}"
 }
@@ -297,7 +298,7 @@ function wait_apiproxy_image() {
 }
 
 function download_client_binaries() {
-  gsutil -m cp "gs://apiproxy-testing-presubmit-binaries/*" ${ROOT}/bin/
+  gcloud storage cp "gs://apiproxy-testing-presubmit-binaries/*" ${ROOT}/bin/
   mv ${ROOT}/bin/api_descriptor.pb ${ROOT}/tests/endpoints/grpc_echo/proto/api_descriptor.pb
   chmod +x ${ROOT}/bin/*
 }