From 95528aeab9135860dab69ef545e16ffab0221bd5 Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Thu, 25 Dec 2025 04:21:22 +0000 Subject: [PATCH] Migrate gsutil usage to gcloud storage --- Makefile | 2 +- prow/gcpproxy-coverage.sh | 4 ++-- scripts/all-utilities.sh | 8 ++++---- scripts/release/release-qualify.sh | 2 +- src/go/gcsrunner/main/runner.go | 2 +- tests/e2e/scripts/linux-test-kb-long-run.sh | 2 +- tests/e2e/scripts/prow-utilities.sh | 5 +++-- 7 files changed, 13 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 7405e9ef5..26b2b9609 100644 --- a/Makefile +++ b/Makefile @@ -132,7 +132,7 @@ upload-e2e-client-binaries: build-grpc-echo build-grpc-interop @cp -n bin/stress_test /tmp/apiproxy-test-presubmit-binaries/stress_test @cp -n bin/grpc_echo_client /tmp/apiproxy-test-presubmit-binaries/grpc_echo_client @cp -n tests/endpoints/grpc_echo/proto/api_descriptor.pb /tmp/apiproxy-test-presubmit-binaries/api_descriptor.pb - @gsutil cp -Z -r /tmp/apiproxy-test-presubmit-binaries/* "gs://apiproxy-testing-presubmit-binaries/" + @gcloud storage cp --gzip-local-all --recursive /tmp/apiproxy-test-presubmit-binaries/* "gs://apiproxy-testing-presubmit-binaries/" #----------------------------------------------------------------------------- diff --git a/prow/gcpproxy-coverage.sh b/prow/gcpproxy-coverage.sh index 98423b46b..f6661f370 100755 --- a/prow/gcpproxy-coverage.sh +++ b/prow/gcpproxy-coverage.sh @@ -73,10 +73,10 @@ case "${JOB_TYPE}" in esac # Upload folder. -gsutil -m rsync -r -d "${ROOT}/generated" "gs://esp-v2-coverage/${PUBLIC_DIRECTORY}" +gcloud storage rsync --recursive --delete-unmatched-destination-objects "${ROOT}/generated" "gs://esp-v2-coverage/${PUBLIC_DIRECTORY}" # No browser cache since some directories change often and that would be misleading. -gsutil -m setmeta -h "Cache-Control:private, max-age=0, no-transform" "gs://esp-v2-coverage/${PUBLIC_DIRECTORY}/**" +gcloud storage objects update --cache-control="private, max-age=0, no-transform" "gs://esp-v2-coverage/${PUBLIC_DIRECTORY}/**" echo '=======================================================' echo '==================== View Coverage ====================' diff --git a/scripts/all-utilities.sh b/scripts/all-utilities.sh index f30e2fc66..6794e4cf4 100755 --- a/scripts/all-utilities.sh +++ b/scripts/all-utilities.sh @@ -131,7 +131,7 @@ function retry() { # Download api Keys from Cloud storage and source the file. function set_api_keys() { local api_key_directory="$(mktemp -d)" - $GSUTIL cp gs://apiproxy-testing-client-secret-files/api_keys \ + gcloud storage cp gs://apiproxy-testing-client-secret-files/api_keys \ "${api_key_directory}/api_keys" \ || error_exit "Failed to download API key file." @@ -142,7 +142,7 @@ function set_api_keys() { function get_test_client_key() { local remote_file_name=$1 local key_path=$2 - $GSUTIL cp "gs://apiproxy-testing-client-secret-files/$remote_file_name" "$key_path" + gcloud storage cp "gs://apiproxy-testing-client-secret-files/$remote_file_name" "$key_path" echo -n "$key_path" return 0 } @@ -279,7 +279,7 @@ function update_tool() { [[ -z "${TOOLS_BUCKET}" ]] && return 1 echo "Uploading ${local_path} to ${remote_path}." - ${GSUTIL} cp "${local_path}" "${remote_path}" \ + gcloud storage cp "${local_path}" "${remote_path}" \ || { echo "Failed to upload ${tool_name} to ${TOOLS_BUCKET}"; return 1; } return 0 @@ -293,7 +293,7 @@ function get_tool() { [[ -z "${TOOLS_BUCKET}" ]] && return 1 echo "Downloading ${remote_path} to ${local_path}." - ${GSUTIL} cp "${remote_path}" "${local_path}" \ + gcloud storage cp "${remote_path}" "${local_path}" \ || { echo "Failed to upload ${tool_name} to ${TOOLS_BUCKET}"; return 1; } return 0 diff --git a/scripts/release/release-qualify.sh b/scripts/release/release-qualify.sh index 16f4bb2c0..803a8d2e5 100755 --- a/scripts/release/release-qualify.sh +++ b/scripts/release/release-qualify.sh @@ -114,7 +114,7 @@ function count_stress_failures() { mkdir -p "${LOGS}/${SHA}" echo "Downloading prow logs to '${LOGS}' directory." - ${GSUTIL} -m -q cp -r "gs://apiproxy-continuous-long-run/${SHA}/logs/*" "${LOGS}/${SHA}/" 2>&1 \ + gcloud -q storage cp --recursive "gs://apiproxy-continuous-long-run/${SHA}/logs/*" "${LOGS}/${SHA}/" 2>&1 \ || error_exit "Failed to download logs from gs://apiproxy-continuous-long-run/${SHA}/logs/*" python3 "${ROOT}/scripts/release/validate_release.py" \ diff --git a/src/go/gcsrunner/main/runner.go b/src/go/gcsrunner/main/runner.go index 8012931da..15d66e16f 100644 --- a/src/go/gcsrunner/main/runner.go +++ b/src/go/gcsrunner/main/runner.go @@ -21,7 +21,7 @@ // Fetches from the bucket at `BUCKET` at the path `CONFIG_FILE_NAME`, // Behavior is similar to running: // -// `gsutil cp "gs://${BUCKET}/${CONFIG_FILE_NAME}" envoy.json` +// `gcloud storage cp "gs://${BUCKET}/${CONFIG_FILE_NAME}" envoy.json` // // without needing `gsutil` in the image. // diff --git a/tests/e2e/scripts/linux-test-kb-long-run.sh b/tests/e2e/scripts/linux-test-kb-long-run.sh index 80457f004..8ec9f7c27 100755 --- a/tests/e2e/scripts/linux-test-kb-long-run.sh +++ b/tests/e2e/scripts/linux-test-kb-long-run.sh @@ -56,7 +56,7 @@ fi # Download api Keys with restrictions from Cloud storage. TEMP_DIR="$(mktemp -d)" API_RESTRICTION_KEYS_FILE="${TEMP_DIR}/apiproxy-e2e-key-restriction.json" -gsutil cp gs://apiproxy-testing-client-secret-files/restricted_api_keys.json \ +gcloud storage cp gs://apiproxy-testing-client-secret-files/restricted_api_keys.json \ "${API_RESTRICTION_KEYS_FILE}" \ || error_exit "Failed to download API key with restrictions file." diff --git a/tests/e2e/scripts/prow-utilities.sh b/tests/e2e/scripts/prow-utilities.sh index 5287b4987..11943d671 100755 --- a/tests/e2e/scripts/prow-utilities.sh +++ b/tests/e2e/scripts/prow-utilities.sh @@ -257,7 +257,8 @@ function upload_logs() { local log_dir="${2}" echo "Uploading content of ${log_dir} to ${remote_dir}" - retry -n 3 ${GSUTIL} -h 'Content-Type:text/plain' -m cp -r \ + # The gsutil top-level flag `-h` is not supported in `gcloud storage`. + retry -n 3 gcloud storage cp --recursive \ "${log_dir}" "${remote_dir}" \ || echo "Failed to upload ${log_dir}" } @@ -297,7 +298,7 @@ function wait_apiproxy_image() { } function download_client_binaries() { - gsutil -m cp "gs://apiproxy-testing-presubmit-binaries/*" ${ROOT}/bin/ + gcloud storage cp "gs://apiproxy-testing-presubmit-binaries/*" ${ROOT}/bin/ mv ${ROOT}/bin/api_descriptor.pb ${ROOT}/tests/endpoints/grpc_echo/proto/api_descriptor.pb chmod +x ${ROOT}/bin/* }