Storage batch operation samples v2

google-cloud-storage_batch_operations/samples/.rubocop.yml

@@ -0,0 +1,24 @@
+inherit_gem:
+    google-style: google-style.yml
+
+Lint/EmptyBlock:
+    Enabled: false
+Lint/ShadowingOuterLocalVariable:
+    Enabled: false
+Metrics/AbcSize:
+    Enabled: false
+Metrics/BlockLength:
+    Enabled: false
+Metrics/CyclomaticComplexity:
+    Max: 12
+Metrics/MethodLength:
+    Enabled: false
+Naming/AccessorMethodName:
+    Enabled: false
+Style/GlobalVars:
+    Exclude:
+        - "acceptance/helper.rb"
+Style/Next:
+    Enabled: false
+Style/SymbolProc:
+    Enabled: false

google-cloud-storage_batch_operations/samples/Gemfile

@@ -0,0 +1,30 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source "https://rubygems.org"
+
+if ENV["GOOGLE_CLOUD_SAMPLES_TEST"] == "master"
+  gem "google-cloud-storage", group: :test, path: "../../google-cloud-storage"
+  gem "google-cloud-storage_batch_operations", group: :test, path: "../../google-cloud-storage_batch_operations"
+else
+  gem "google-cloud-storage"
+  gem "google-cloud-storage_batch_operations"
+end
+group :test do
+  gem "google-style", "~> 1.30.0"
+  gem "minitest", "~> 5.14"
+  gem "minitest-focus", "~> 1.1"
+  gem "minitest-hooks", "~> 1.5"
+  gem "rake"
+end

google-cloud-storage_batch_operations/samples/Rakefile

@@ -0,0 +1,20 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "rake/testtask"
+
+Rake::TestTask.new "test" do |t|
+  t.test_files = FileList["**/*_test.rb"]
+  t.warning = false
+end

google-cloud-storage_batch_operations/samples/acceptance/batch_job_test.rb

@@ -0,0 +1,90 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require_relative "helper"
+require "google/cloud/storage"
+require_relative "../storage_batch_create_job"
+require_relative "../storage_batch_delete_job"
+require_relative "../storage_batch_cancel_job"
+require_relative "../storage_batch_list_job"
+require_relative "../storage_batch_get_job"
+require 'pry'
+
+describe "Batch jobs Snippets" do
+  let(:storage)   {  Google::Cloud::Storage.new}
+  let(:project_name)   { storage.project }
+  let(:bucket)         { @bucket }
+  let(:file_content)   { "some content" }
+  let(:remote_file_name) { "ruby_file_#{SecureRandom.hex}" }
+
+  before :all do
+    @bucket = create_bucket_helper random_bucket_name
+  end
+
+  after :all do
+    delete_bucket_helper @bucket.name
+  end
+
+  def create_test_job my_job
+    bucket.create_file StringIO.new(file_content), remote_file_name
+    create_job bucket_name: bucket.name, prefix: "ruby_file", job_name: my_job, project_name: project_name
+  end
+
+  describe "storage batch manage operations" do
+    before do
+      @job_name = "ruby-sbo-job-#{SecureRandom.hex}"
+      create_test_job @job_name
+    end
+
+    it "lists jobs and includes the created job" do
+      out, _err = capture_io { list_job project_name: project_name }
+      assert_includes out, @job_name, "Expected job name not found in the result list"
+    end
+
+    it "fetches the details of a job" do
+      result = get_job project_name: project_name, job_name: @job_name
+      assert_includes result, @job_name, "Expected job name not found in the result"
+    end
+
+    it "cancels a job" do
+      assert_output "The #{@job_name} is canceled.\n" do
+        cancel_job project_name: project_name, job_name: @job_name
+      end
+    end
+  end
+
+  describe "Delete storage batch operation" do
+    before do
+      @job_name = "ruby-sbo-job-#{SecureRandom.hex}"
+      create_test_job @job_name
+    end
+    it "deletes a job" do
+      retry_job_status do
+        get_job project_name: project_name, job_name: @job_name
+      end
+      assert_output "The #{@job_name} is deleted.\n" do
+        delete_job project_name: project_name, job_name: @job_name
+      end
+    end
+  end
+
+  describe "creates a storage batch operation" do
+    it "creates a job" do
+      @job_name = "ruby-sbo-job-#{SecureRandom.hex}"
+      assert_output "The #{@job_name} is created.\n" do
+        create_test_job @job_name
+      end
+    end
+  end
+end

google-cloud-storage_batch_operations/samples/acceptance/helper.rb

@@ -0,0 +1,68 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "google/cloud/errors"
+require "google/cloud/storage"
+require "minitest/autorun"
+require "minitest/focus"
+require "minitest/hooks/default"
+require "net/http"
+require "time"
+require "securerandom"
+
+def create_bucket_helper bucket_name
+  storage_client = Google::Cloud::Storage.new
+  retry_resource_exhaustion do
+    storage_client.create_bucket bucket_name
+  end
+end
+
+def delete_bucket_helper bucket_name
+  storage_client = Google::Cloud::Storage.new
+  retry_resource_exhaustion do
+    bucket = storage_client.bucket bucket_name
+    return unless bucket
+
+    bucket.files.each(&:delete)
+    bucket.delete
+  end
+end
+
+def retry_job_status
+  5.times do
+    status = yield[/job_status-\s*(\w+)/, 1]
+    break unless status == "RUNNING"
+
+    puts "Job in Running status. Gonna try again"
+    sleep rand(10..16)
+  end
+end
+
+def retry_resource_exhaustion
+  5.times do
+    return yield
+  rescue Google::Cloud::ResourceExhaustedError => e
+    puts "\n#{e} Gonna try again"
+    sleep rand(10..16)
+  rescue StandardError => e
+    puts "\n#{e}"
+    raise e
+  end
+  raise Google::Cloud::ResourceExhaustedError, "Maybe take a break from creating and deleting buckets for a bit"
+end
+
+def random_bucket_name
+  t = Time.now.utc.iso8601.gsub ":", "-"
+  "ruby-storage-samples-test-#{t}-#{SecureRandom.hex 4}".downcase
+end

google-cloud-storage_batch_operations/samples/storage_batch_cancel_job.rb

@@ -0,0 +1,38 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require "google/cloud/storage_batch_operations"
+
+# [START storage_batch_cancel_job]
+def cancel_job project_name:, job_name:
+  # The Name/ID of your project
+  # project_name = "your-project-id"
+
+  # The name of your Storage batch operation job
+  # job_name = "your-job-name"
+
+  client = Google::Cloud::StorageBatchOperations.storage_batch_operations
+  parent = "projects/#{project_name}/locations/global"
+
+  request = Google::Cloud::StorageBatchOperations::V1::CancelJobRequest.new name: "#{parent}/jobs/#{job_name}"
+  result = client.cancel_job request
+  message = if result.is_a? Google::Cloud::StorageBatchOperations::V1::CancelJobResponse
+              "The #{job_name} is canceled."
+            else
+              "The #{job_name} is not canceled."
+            end
+  puts message
+end
+# [END storage_batch_cancel_job]
+
+cancel_job project_name: ARGV.shift, job_name: ARGV.shift if $PROGRAM_NAME == __FILE__

google-cloud-storage_batch_operations/samples/storage_batch_create_job.rb

@@ -0,0 +1,72 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require "google/cloud/storage_batch_operations"
+
+# [START storage_batch_create_job]
+def create_job bucket_name:, prefix:, job_name:, project_name:
+  # The ID of your GCS bucket
+  # bucket_name = "your-unique-bucket-name"
+
+  # Prefix is the first part of filename on which job has to be executed
+  # prefix = 'test'
+
+  # The Name/ID of your project
+  # project_name = "your-project-id"
+
+  # The name of your Storage batch operation job
+  # job_name = "your-job-name"
+
+  client = Google::Cloud::StorageBatchOperations.storage_batch_operations
+
+  parent = "projects/#{project_name}/locations/global"
+
+  prefix_list = Google::Cloud::StorageBatchOperations::V1::PrefixList.new(
+    included_object_prefixes: [prefix]
+  )
+
+  bucket = Google::Cloud::StorageBatchOperations::V1::BucketList::Bucket.new(
+    bucket: bucket_name,
+    prefix_list: prefix_list
+  )
+
+  bucket_list = Google::Cloud::StorageBatchOperations::V1::BucketList.new(
+    buckets: [bucket]
+  )
+
+  # Define the delete operation
+  delete_object = Google::Cloud::StorageBatchOperations::V1::DeleteObject.new(
+    permanent_object_deletion_enabled: false
+  )
+
+  # Build the job
+  job = Google::Cloud::StorageBatchOperations::V1::Job.new(
+    name: job_name,
+    bucket_list: bucket_list,
+    delete_object: delete_object
+  )
+
+  request = Google::Cloud::StorageBatchOperations::V1::CreateJobRequest.new parent: parent, job_id: job_name, job: job
+  result = client.create_job request
+
+  puts result.is_a?(Gapic::Operation) ? "The #{job_name} is created." : "The #{job_name} is not created."
+end
+# [END storage_batch_create_job]
+
+if $PROGRAM_NAME == __FILE__
+  create_job(
+    bucket_name: ARGV.shift,
+    prefix: ARGV.shift,
+    job_name: ARGV.shift,
+    project_name: ARGV.shift  )
+end

google-cloud-storage_batch_operations/samples/storage_batch_delete_job.rb

@@ -0,0 +1,32 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require "google/cloud/storage_batch_operations"
+
+# [START storage_batch_delete_job]
+def delete_job project_name:, job_name:
+  # The Name/ID of your project
+  # project_name = "your-project-id"
+
+  # The name of your Storage batch operation job
+  # job_name = "your-job-name"
+
+  client = Google::Cloud::StorageBatchOperations.storage_batch_operations
+  parent = "projects/#{project_name}/locations/global"
+  request = Google::Cloud::StorageBatchOperations::V1::DeleteJobRequest.new name: "#{parent}/jobs/#{job_name}"
+  result = client.delete_job request
+  puts result.is_a?(Google::Protobuf::Empty) ? "The #{job_name} is deleted." : "The #{job_name} is not deleted."
+end
+# [END storage_batch_delete_job]
+
+delete_job project_name: ARGV.shift, job_name: ARGV.shift if $PROGRAM_NAME == __FILE__