Skip to content

Commit 054cff2

Browse files
shubhangi-googleshubhangi-google
authored
samples(storage batch operations): Add batch operations jobs samples (#30511)
* changes working * fixing lint * updating * updating * fixing test cases * fix test cases * fix comments * fix test * Update Gemfile * Update batch_job_test.rb * Update batch_job_test.rb * Update helper.rb * adding full License * fixing lint * fixing lint * using wrapper class * refactoring * resolving syntax error * refactoring * refactoring * Update storage_batch_cancel_job.rb * refactoring * Update storage_batch_list_job.rb * Update storage_batch_list_job.rb * Update batch_job_test.rb * refactor * lint fix * Updated test case description * fix test case * refactor * fix lint * fix lint
1 parent 3d019bf commit 054cff2

File tree

10 files changed

+476
-0
lines changed

10 files changed

+476
-0
lines changed

‎google-cloud-storage_batch_operations/samples/.rubocop.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
inherit_gem:
2+
google-style: google-style.yml
3+
4+
Lint/EmptyBlock:
5+
Enabled: false
6+
Lint/ShadowingOuterLocalVariable:
7+
Enabled: false
8+
Metrics/AbcSize:
9+
Enabled: false
10+
Metrics/BlockLength:
11+
Enabled: false
12+
Metrics/CyclomaticComplexity:
13+
Max: 12
14+
Metrics/MethodLength:
15+
Enabled: false
16+
Naming/AccessorMethodName:
17+
Enabled: false
18+
Style/GlobalVars:
19+
Exclude:
20+
- "acceptance/helper.rb"
21+
Style/Next:
22+
Enabled: false
23+
Style/SymbolProc:
24+
Enabled: false

‎google-cloud-storage_batch_operations/samples/Gemfile‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
source "https://rubygems.org"
16+
17+
if ENV["GOOGLE_CLOUD_SAMPLES_TEST"] == "master"
18+
gem "google-cloud-storage", group: :test, path: "../../google-cloud-storage"
19+
gem "google-cloud-storage_batch_operations", group: :test, path: "../../google-cloud-storage_batch_operations"
20+
else
21+
gem "google-cloud-storage"
22+
gem "google-cloud-storage_batch_operations"
23+
end
24+
group :test do
25+
gem "google-style", "~> 1.30.0"
26+
gem "minitest", "~> 5.14"
27+
gem "minitest-focus", "~> 1.1"
28+
gem "minitest-hooks", "~> 1.5"
29+
gem "rake"
30+
end

‎google-cloud-storage_batch_operations/samples/Rakefile‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
require "rake/testtask"
16+
17+
Rake::TestTask.new "test" do |t|
18+
t.test_files = FileList["**/*_test.rb"]
19+
t.warning = false
20+
end

‎google-cloud-storage_batch_operations/samples/acceptance/batch_job_test.rb‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
require_relative "helper"
16+
require_relative "../storage_batch_create_job"
17+
require_relative "../storage_batch_delete_job"
18+
require_relative "../storage_batch_cancel_job"
19+
require_relative "../storage_batch_list_jobs"
20+
require_relative "../storage_batch_get_job"
21+
22+
describe "Storage Batch Operations" do
23+
let(:bucket_name) { random_bucket_name }
24+
let(:project_id) { storage_client.project }
25+
let(:file_content) { "some content" }
26+
let(:remote_file_name) { "ruby_file_#{SecureRandom.hex}" }
27+
let(:job_name_prefix) { "projects/#{project_id}/locations/global/jobs/" }
28+
29+
before :all do
30+
bucket = create_bucket_helper bucket_name
31+
bucket.create_file StringIO.new(file_content), remote_file_name
32+
end
33+
34+
after :all do
35+
delete_bucket_helper bucket_name
36+
end
37+
38+
it "handles Storage batch operation lifecycle in sequence" do
39+
job_id = "ruby-sbo-job-#{SecureRandom.hex}"
40+
job_name = "#{job_name_prefix}#{job_id}"
41+
42+
# Create job
43+
assert_output(/Storage Batch Operations job #{job_name} is created./) do
44+
create_job bucket_name: bucket_name, prefix: "ruby_file", job_id: job_id, project_id: project_id
45+
end
46+
47+
# List jobs
48+
assert_output(/Job name: #{job_name} present in the list/) do
49+
list_jobs project_id: project_id
50+
end
51+
52+
# Get job details
53+
assert_output(/Storage Batch Operations job Found - #{job_name}, job_status- /) do
54+
get_job project_id: project_id, job_id: job_id
55+
end
56+
57+
# Cancel job
58+
expected_output_pattern = /Storage Batch Operations job #{job_name} (is canceled|was already completed)\./
59+
assert_output expected_output_pattern do
60+
cancel_job project_id: project_id, job_id: job_id
61+
end
62+
63+
# Delete job
64+
assert_output "The #{job_id} is deleted.\n" do
65+
delete_job project_id: project_id, job_id: job_id
66+
end
67+
end
68+
end

‎google-cloud-storage_batch_operations/samples/acceptance/helper.rb‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
require "google/cloud/errors"
16+
require "google/cloud/storage"
17+
require "minitest/autorun"
18+
require "minitest/focus"
19+
require "minitest/hooks/default"
20+
require "net/http"
21+
require "time"
22+
require "securerandom"
23+
24+
def create_bucket_helper bucket_name
25+
retry_resource_exhaustion do
26+
storage_client.create_bucket bucket_name
27+
end
28+
end
29+
30+
def storage_client
31+
@storage_client ||= Google::Cloud::Storage.new
32+
end
33+
34+
def delete_bucket_helper bucket_name
35+
retry_resource_exhaustion do
36+
bucket = storage_client.bucket bucket_name
37+
return unless bucket
38+
39+
bucket.files.each(&:delete)
40+
bucket.delete
41+
end
42+
end
43+
44+
def retry_resource_exhaustion
45+
attempts = 5
46+
start_time = Time.now
47+
last_error = nil
48+
49+
attempts.times do |i|
50+
begin
51+
return yield
52+
rescue StandardError => e
53+
last_error = e
54+
puts "\nAttempt #{i + 1} failed with #{e.class}. Retrying..."
55+
sleep rand(10..16)
56+
end
57+
end
58+
59+
elapsed_time = Time.now - start_time
60+
message = "Failed after #{attempts} attempts in #{elapsed_time.round 2} seconds. " \
61+
"Last error: #{last_error.message}"
62+
raise last_error, message, last_error.backtrace
63+
end
64+
65+
def random_bucket_name
66+
t = Time.now.utc.iso8601.gsub ":", "-"
67+
"ruby-sbo-samples-test-#{t}-#{SecureRandom.hex 4}".downcase
68+
end

‎google-cloud-storage_batch_operations/samples/storage_batch_cancel_job.rb‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# [START storage_batch_cancel_job]
16+
require "google/cloud/storage_batch_operations"
17+
18+
# Cancels a Storage Batch Operations job.
19+
#
20+
# Note: If the job is already completed or does not exist, a message indicating
21+
# this will be printed instead of raising an error.
22+
#
23+
# @param project_id [String] The ID of your Google Cloud project.
24+
# @param job_id [String] The ID of the Storage Batch Operations job to cancel.
25+
#
26+
# @example
27+
# cancel_job project_id: "your-project-id", job_id: "your-job-id"
28+
#
29+
def cancel_job project_id:, job_id:
30+
client = Google::Cloud::StorageBatchOperations.storage_batch_operations
31+
parent = "projects/#{project_id}/locations/global"
32+
33+
request = Google::Cloud::StorageBatchOperations::V1::CancelJobRequest.new name: "#{parent}/jobs/#{job_id}"
34+
35+
# To fetch job details using get_job
36+
get_request = Google::Cloud::StorageBatchOperations::V1::GetJobRequest.new name: "#{parent}/jobs/#{job_id}"
37+
38+
begin
39+
result = client.cancel_job request
40+
## Fetch the job
41+
job_detail = client.get_job get_request
42+
message = "Storage Batch Operations job #{job_detail.name} is canceled."
43+
rescue Google::Cloud::FailedPreconditionError
44+
## Fetch the job
45+
job_detail = client.get_job get_request
46+
# This error is thrown when the job is already completed.
47+
message = "Storage Batch Operations job #{job_detail.name} was already completed."
48+
rescue StandardError
49+
# This error is thrown when the job is not canceled.
50+
message = "Failed to cancel job #{job_id}. Error: #{result.error.message}"
51+
end
52+
puts message
53+
end
54+
# [END storage_batch_cancel_job]
55+
56+
cancel_job project_id: ARGV.shift, job_id: ARGV.shift if $PROGRAM_NAME == __FILE__

‎google-cloud-storage_batch_operations/samples/storage_batch_create_job.rb‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# [START storage_batch_create_job]
16+
require "google/cloud/storage_batch_operations"
17+
18+
# Creates a Storage Batch Operations job to delete objects in a bucket
19+
# that match a given prefix. The deletion is a "soft delete", meaning
20+
# objects can be recovered if versioning is enabled on the bucket.
21+
#
22+
# @param bucket_name [String] The name of the Google Cloud Storage bucket.
23+
# @param prefix [String] The prefix of the objects to be included in the job.
24+
# The job will operate on all objects whose names start with this prefix.
25+
# @param job_id [String] A unique identifier for the job.
26+
# @param project_id [String] The ID of the Google Cloud project where the job will be created.
27+
#
28+
# @example
29+
# create_job(
30+
# bucket_name: "your-unique-bucket-name",
31+
# prefix: "test-files/",
32+
# job_id: "your-job-id",
33+
# project_id: "your-project-id"
34+
# )
35+
#
36+
def create_job bucket_name:, prefix:, job_id:, project_id:
37+
client = Google::Cloud::StorageBatchOperations.storage_batch_operations
38+
39+
parent = "projects/#{project_id}/locations/global"
40+
41+
prefix_list = Google::Cloud::StorageBatchOperations::V1::PrefixList.new(
42+
included_object_prefixes: [prefix]
43+
)
44+
45+
bucket = Google::Cloud::StorageBatchOperations::V1::BucketList::Bucket.new(
46+
bucket: bucket_name,
47+
prefix_list: prefix_list
48+
)
49+
50+
bucket_list = Google::Cloud::StorageBatchOperations::V1::BucketList.new(
51+
buckets: [bucket]
52+
)
53+
54+
# Define the delete operation
55+
delete_object = Google::Cloud::StorageBatchOperations::V1::DeleteObject.new(
56+
permanent_object_deletion_enabled: false
57+
)
58+
59+
# Build the job
60+
job = Google::Cloud::StorageBatchOperations::V1::Job.new(
61+
bucket_list: bucket_list,
62+
delete_object: delete_object
63+
)
64+
65+
request = Google::Cloud::StorageBatchOperations::V1::CreateJobRequest.new parent: parent, job_id: job_id, job: job
66+
create_job_operation = client.create_job request
67+
68+
# To fetch job details using get_job to confirm creation
69+
get_request = Google::Cloud::StorageBatchOperations::V1::GetJobRequest.new name: "#{parent}/jobs/#{job_id}"
70+
71+
begin
72+
## Waiting for operation to complete
73+
create_job_operation.wait_until_done!
74+
## Fetch the newly created job to confirm creation
75+
job_detail = client.get_job get_request
76+
message = "Storage Batch Operations job #{job_detail.name} is created."
77+
rescue StandardError
78+
# This error is thrown when the job is not created.
79+
message = "Failed to create job #{job_id}. Error: #{create_job_operation.error.message}"
80+
end
81+
puts message
82+
end
83+
# [END storage_batch_create_job]
84+
85+
if $PROGRAM_NAME == __FILE__
86+
create_job(
87+
bucket_name: ARGV.shift,
88+
prefix: ARGV.shift,
89+
job_id: ARGV.shift,
90+
project_id: ARGV.shift
91+
)
92+
end

0 commit comments

Comments
 (0)