Skip to content

Commit 08d52f7

Browse files
bhandarivijay-pngbhandarivijay-png
committed
Migrate gsutil usage to gcloud storage
1 parent 57e2a04 commit 08d52f7

File tree

7 files changed

+36
-36
lines changed

7 files changed

+36
-36
lines changed

‎.ci/magician/cmd/vcr_cassette_update.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -123,8 +123,8 @@ func execVCRCassetteUpdate(buildID, today string, rnr ExecRunner, ctlr *source.C
123123
// main cassettes backup
124124
// incase nightly run goes wrong. this will be used to restore the cassettes
125125
cassettePath := vt.CassettePath(provider.Beta)
126-
args := []string{"-m", "-q", "cp", filepath.Join(cassettePath, "*"), bucketPrefix + "/main_cassettes_backup/fixtures/"}
127-
if _, err := rnr.Run("gsutil", args, nil); err != nil {
126+
args := []string{"storage", "cp", filepath.Join(cassettePath, "*"), bucketPrefix + "/main_cassettes_backup/fixtures/"}
127+
if _, err := rnr.Run("gcloud", args, nil); err != nil {
128128
return fmt.Errorf("error backup cassettes: %w", err)
129129
}
130130

@@ -222,18 +222,18 @@ func execVCRCassetteUpdate(buildID, today string, rnr ExecRunner, ctlr *source.C
222222
}
223223

224224
func uploadLogsToGCS(src, dest string, rnr ExecRunner) (string, error) {
225-
return uploadToGCS(src, dest, []string{"-h", "Content-Type:text/plain", "-q", "cp", "-r"}, rnr)
225+
return uploadToGCS(src, dest, []string{"storage", "cp", "--content-type=text/plain", "--recursive"}, rnr)
226226
}
227227

228228
func uploadCassettesToGCS(src, dest string, rnr ExecRunner) (string, error) {
229-
return uploadToGCS(src, dest, []string{"-m", "-q", "cp"}, rnr)
229+
return uploadToGCS(src, dest, []string{"storage", "cp"}, rnr)
230230
}
231231

232232
func uploadToGCS(src, dest string, opts []string, rnr ExecRunner) (string, error) {
233233
fmt.Printf("uploading from %s to %s\n", src, dest)
234234
args := append(opts, src, dest)
235-
fmt.Println("gsutil", args)
236-
return rnr.Run("gsutil", args, nil)
235+
fmt.Println("gcloud", args)
236+
return rnr.Run("gcloud", args, nil)
237237
}
238238

239239
func formatVCRCassettesUpdateReplaying(data vcrCassetteUpdateReplayingResult) (string, error) {

‎.ci/magician/cmd/vcr_merge.go‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ var vcrMergeCmd = &cobra.Command{
2020
2121
It then performs the following operations:
2222
1. Get the latest closed PR matching the reference commit SHA.
23-
2. Run gsutil to list, copy, and remove the vcr cassettes fixtures.
23+
2. Run gcloud storage to list, copy, and remove the vcr cassettes fixtures.
2424
`,
2525
Args: cobra.ExactArgs(1),
2626
RunE: func(cmd *cobra.Command, args []string) error {
@@ -88,11 +88,12 @@ func mergeCassettes(basePath, baseBranch, prPath string, runner source.Runner) {
8888

8989
func listCassettes(path string, runner source.Runner) error {
9090
lsArgs := []string{
91+
"storage",
9192
"ls",
9293
path,
9394
}
94-
fmt.Println("Running command: ", "gsutil", lsArgs)
95-
ret, err := runner.Run("gsutil", lsArgs, nil)
95+
fmt.Println("Running command: ", "gcloud", lsArgs)
96+
ret, err := runner.Run("gcloud", lsArgs, nil)
9697
if err != nil {
9798
return err
9899
}
@@ -102,26 +103,26 @@ func listCassettes(path string, runner source.Runner) error {
102103

103104
func cpCassettes(src, dest string, runner source.Runner) {
104105
cpArgs := []string{
105-
"-m",
106+
"storage",
106107
"cp",
107108
src,
108109
dest,
109110
}
110-
fmt.Println("Running command: ", "gsutil", cpArgs)
111-
if _, err := runner.Run("gsutil", cpArgs, nil); err != nil {
111+
fmt.Println("Running command: ", "gcloud", cpArgs)
112+
if _, err := runner.Run("gcloud", cpArgs, nil); err != nil {
112113
fmt.Println("Error in copy: ", err)
113114
}
114115
}
115116

116117
func rmCassettes(dest string, runner source.Runner) {
117118
rmArgs := []string{
118-
"-m",
119+
"storage",
119120
"rm",
120-
"-r",
121+
"--recursive",
121122
dest,
122123
}
123-
fmt.Println("Running command: ", "gsutil", rmArgs)
124-
if _, err := runner.Run("gsutil", rmArgs, nil); err != nil {
124+
fmt.Println("Running command: ", "gcloud", rmArgs)
125+
if _, err := runner.Run("gcloud", rmArgs, nil); err != nil {
125126
fmt.Println("Error in remove: ", err)
126127
}
127128
}

‎.ci/magician/cmd/vcr_merge_eap.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ var vcrMergeEapCmd = &cobra.Command{
1818
1. CL number
1919
2020
It then performs the following operations:
21-
1. Run gsutil to list, copy, and remove the vcr cassettes fixtures.
21+
1. Run gcloud storage to list, copy, and remove the vcr cassettes fixtures.
2222
`,
2323
Args: cobra.ExactArgs(1),
2424
RunE: func(cmd *cobra.Command, args []string) error {

‎.ci/magician/cmd/vcr_merge_test.go‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -22,25 +22,25 @@ func TestExecVCRMerge(t *testing.T) {
2222
baseBranch: "main",
2323
commitSha: "sha",
2424
calledMethods: []string{
25-
"gsutil ls gs://ci-vcr-cassettes/refs/heads/auto-pr-123/fixtures/",
26-
"gsutil -m cp gs://ci-vcr-cassettes/refs/heads/auto-pr-123/fixtures/* gs://ci-vcr-cassettes/fixtures/",
27-
"gsutil -m rm -r gs://ci-vcr-cassettes/refs/heads/auto-pr-123/",
28-
"gsutil ls gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/fixtures/",
29-
"gsutil -m cp gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/fixtures/* gs://ci-vcr-cassettes/beta/fixtures/",
30-
"gsutil -m rm -r gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/",
25+
"gcloud storage ls gs://ci-vcr-cassettes/refs/heads/auto-pr-123/fixtures/",
26+
"gcloud storage cp gs://ci-vcr-cassettes/refs/heads/auto-pr-123/fixtures/* gs://ci-vcr-cassettes/fixtures/",
27+
"gcloud storage rm --recursive gs://ci-vcr-cassettes/refs/heads/auto-pr-123/",
28+
"gcloud storage ls gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/fixtures/",
29+
"gcloud storage cp gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/fixtures/* gs://ci-vcr-cassettes/beta/fixtures/",
30+
"gcloud storage rm --recursive gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/",
3131
},
3232
},
3333
{
3434
name: "base branch is not main",
3535
baseBranch: "test-branch",
3636
commitSha: "sha",
3737
calledMethods: []string{
38-
"gsutil ls gs://ci-vcr-cassettes/refs/heads/auto-pr-123/fixtures/",
39-
"gsutil -m cp gs://ci-vcr-cassettes/refs/heads/auto-pr-123/fixtures/* gs://ci-vcr-cassettes/refs/branches/test-branch/fixtures/",
40-
"gsutil -m rm -r gs://ci-vcr-cassettes/refs/heads/auto-pr-123/",
41-
"gsutil ls gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/fixtures/",
42-
"gsutil -m cp gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/fixtures/* gs://ci-vcr-cassettes/beta/refs/branches/test-branch/fixtures/",
43-
"gsutil -m rm -r gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/",
38+
"gcloud storage ls gs://ci-vcr-cassettes/refs/heads/auto-pr-123/fixtures/",
39+
"gcloud storage cp gs://ci-vcr-cassettes/refs/heads/auto-pr-123/fixtures/* gs://ci-vcr-cassettes/refs/branches/test-branch/fixtures/",
40+
"gcloud storage rm --recursive gs://ci-vcr-cassettes/refs/heads/auto-pr-123/",
41+
"gcloud storage ls gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/fixtures/",
42+
"gcloud storage cp gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/fixtures/* gs://ci-vcr-cassettes/beta/refs/branches/test-branch/fixtures/",
43+
"gcloud storage rm --recursive gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/",
4444
},
4545
},
4646
{
@@ -54,8 +54,8 @@ func TestExecVCRMerge(t *testing.T) {
5454
lsReturnedError: true,
5555
baseBranch: "main",
5656
calledMethods: []string{
57-
"gsutil ls gs://ci-vcr-cassettes/refs/heads/auto-pr-123/fixtures/",
58-
"gsutil ls gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/fixtures/",
57+
"gcloud storage ls gs://ci-vcr-cassettes/refs/heads/auto-pr-123/fixtures/",
58+
"gcloud storage ls gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/fixtures/",
5959
},
6060
},
6161
}

‎mmv1/third_party/terraform/.teamcity/components/builds/build_steps.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ fun BuildSteps.saveArtifactsToGCS() {
160160
fi
161161
162162
# Copy logs to GCS
163-
gsutil -m cp %teamcity.build.checkoutDir%/debug* gs://teamcity-logs/${'$'}{FOLDER}/%env.BUILD_NUMBER%/
163+
gcloud storage cp %teamcity.build.checkoutDir%/debug* gs://teamcity-logs/${'$'}{FOLDER}/%env.BUILD_NUMBER%/
164164
165165
# Cleanup
166166
rm google-account.json

‎mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2202,8 +2202,7 @@ resource "google_storage_bucket_object" "init_script" {
22022202
content = <<EOL
22032203
#!/bin/bash
22042204
echo "init action success" >> /tmp/%s
2205-
gsutil cp /tmp/%s ${google_storage_bucket.init_bucket.url}
2206-
EOL
2205+
gcloud storage cp /tmp/%s ${google_storage_bucket.init_bucket.url}EOL
22072206
22082207
}
22092208

‎mmv1/third_party/terraform/services/storage/data_source_storage_object_signed_url_internal_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,9 @@ const fakeCredentials = `{
2424
}
2525
`
2626

27-
// The following values are derived from the output of the `gsutil signurl` command.
27+
// The following values are derived from the output of the `gcloud storage sign-url` command.
2828
// i.e.
29-
// gsutil signurl fake_creds.json gs://tf-test-bucket-6159205297736845881/path/to/file
29+
// gcloud storage sign-url --private-key-file=fake_creds.json gs://tf-test-bucket-6159205297736845881/path/to/file
3030
// URL HTTP Method Expiration Signed URL
3131
// gs://tf-test-bucket-6159205297736845881/path/to/file GET 2016-08-12 14:03:30 https://storage.googleapis.com/tf-test-bucket-6159205297736845881/path/to/file?GoogleAccessId=user@gcp-project.iam.gserviceaccount.com&Expires=1470967410&Signature=JJvE2Jc%2BeoagyS1qRACKBGUkgLkKjw7cGymHhtB4IzzN3nbXDqr0acRWGy0%2BEpZ3HYNDalEYsK0lR9Q0WCgty5I0JKmPIuo9hOYa1xTNH%2B22xiWsekxGV%2FcA9FXgWpi%2BFt7fBmMk4dhDe%2BuuYc7N79hd0FYuSBNW1Wp32Bluoe4SNkNAB%2BuIDd9KqPzqs09UAbBoz2y4WxXOQnRyR8GAfb8B%2FDtv62gYjtmp%2F6%2Fyr6xj7byWKZdQt8kEftQLTQmP%2F17Efjp6p%2BXo71Q0F9IhAFiqWfp3Ij8hHDSebLcVb2ULXyHNNQpHBOhFgALrFW3I6Uc3WciLEOsBS9Ej3EGdTg%3D%3D
3232

0 commit comments

Comments
 (0)