alioss-storage-cli based blobstore client

Author: kathap

lib/cloud_controller/blobstore/client_provider.rb

@@ -6,6 +6,7 @@
 require 'cloud_controller/blobstore/safe_delete_client'
 require 'cloud_controller/blobstore/storage_cli/storage_cli_client'
 require 'cloud_controller/blobstore/storage_cli/azure_storage_cli_client'
+require 'cloud_controller/blobstore/storage_cli/alioss_storage_cli_client'
 require 'google/apis/errors'
 
 module CloudController

lib/cloud_controller/blobstore/storage_cli/alioss_storage_cli_client.rb

@@ -0,0 +1,11 @@
+module CloudController
+  module Blobstore
+    class AliStorageCliClient < StorageCliClient
+      def cli_path
+        ENV['ALI_STORAGE_CLI_PATH'] || '/var/vcap/packages/ali-storage-cli/bin/ali-storage-cli'
+      end
+
+      CloudController::Blobstore::StorageCliClient.register('aliyun', AliStorageCliClient)
+    end
+  end
+end

lib/cloud_controller/blobstore/storage_cli/storage_cli_client.rb

@@ -80,7 +80,11 @@ def validate_required_keys!(json, path)
           provider = json['provider'].to_s.strip
           raise BlobstoreError.new("No provider specified in config file: #{path.inspect}") if provider.empty?
 
-          required = %w[account_key account_name container_name environment]
+          if provider == 'AzureRM'
+            required = %w[account_key account_name container_name environment]
+          elsif provider == 'aliyun'
+            required = %w[access_key_id access_key_secret endpoint bucket_name]
+          end
           missing = required.reject { |k| json.key?(k) && !json[k].to_s.strip.empty? }
           return if missing.empty?
 

spec/unit/lib/cloud_controller/blobstore/storage_cli/alioss_storage_cli_client_spec.rb

@@ -0,0 +1,215 @@
+require 'spec_helper'
+require 'tempfile'
+require 'json'
+require_relative '../client_shared'
+require 'cloud_controller/blobstore/storage_cli/azure_storage_cli_client'
+require 'cloud_controller/blobstore/storage_cli/storage_cli_blob'
+
+module CloudController
+  module Blobstore
+    RSpec.describe AzureStorageCliClient do
+      let!(:tmp_cfg) do
+        f = Tempfile.new(['storage_cli_config', '.json'])
+        f.write({ provider: 'AzureRM',
+                  account_name: 'some-account-name',
+                  account_key: 'some-access-key',
+                  container_name: directory_key,
+                  environment: 'AzureCloud' }.to_json)
+        f.flush
+        f
+      end
+
+      before do
+        cc_cfg = instance_double(VCAP::CloudController::Config)
+        allow(VCAP::CloudController::Config).to receive(:config).and_return(cc_cfg)
+
+        allow(cc_cfg).to receive(:get) do |key, *_|
+          case key
+          when :storage_cli_config_file_droplets,
+            :storage_cli_config_file_buildpacks,
+            :storage_cli_config_file_packages,
+            :storage_cli_config_file_resource_pool
+            tmp_cfg.path
+          end
+        end
+        allow(Steno).to receive(:logger).and_return(double(info: nil, error: nil))
+      end
+
+      after { tmp_cfg.close! }
+
+      subject(:client) { AzureStorageCliClient.new(provider: 'AzureRM', directory_key: directory_key, resource_type: resource_type, root_dir: 'bommel', config_path: 'path') }
+      let(:directory_key) { 'my-bucket' }
+      let(:resource_type) { 'resource_pool' }
+      let(:downloaded_file) do
+        Tempfile.open('') do |tmpfile|
+          tmpfile.write('downloaded file content')
+          tmpfile
+        end
+      end
+
+      let(:deletable_blob) { StorageCliBlob.new('deletable-blob') }
+      let(:dest_path) { File.join(Dir.mktmpdir, SecureRandom.uuid) }
+
+      describe 'conforms to the blobstore client interface' do
+        before do
+          allow(client).to receive(:run_cli).with('exists', anything, allow_exit_code_three: true).and_return([nil, instance_double(Process::Status, exitstatus: 0)])
+          allow(client).to receive(:run_cli).with('get', anything, anything).and_wrap_original do |_original_method, _cmd, _source, dest_path|
+            File.write(dest_path, 'downloaded content')
+            [nil, instance_double(Process::Status, exitstatus: 0)]
+          end
+          allow(client).to receive(:run_cli).with('put', anything, anything).and_return([nil, instance_double(Process::Status, exitstatus: 0)])
+          allow(client).to receive(:run_cli).with('copy', anything, anything).and_return([nil, instance_double(Process::Status, exitstatus: 0)])
+          allow(client).to receive(:run_cli).with('delete', anything).and_return([nil, instance_double(Process::Status, exitstatus: 0)])
+          allow(client).to receive(:run_cli).with('delete-recursive', anything).and_return([nil, instance_double(Process::Status, exitstatus: 0)])
+          allow(client).to receive(:run_cli).with('list', anything).and_return(["aa/bb/blob1\ncc/dd/blob2\n", instance_double(Process::Status, exitstatus: 0)])
+          allow(client).to receive(:run_cli).with('ensure-bucket-exists').and_return([nil, instance_double(Process::Status, exitstatus: 0)])
+          allow(client).to receive(:run_cli).with('properties', anything).and_return(['{"dummy": "json"}', instance_double(Process::Status, exitstatus: 0)])
+          allow(client).to receive(:run_cli).with('sign', anything, 'get', '3600s').and_return(['some-url', instance_double(Process::Status, exitstatus: 0)])
+        end
+
+        it_behaves_like 'a blobstore client'
+      end
+
+      describe '#local?' do
+        it 'returns false' do
+          expect(client.local?).to be false
+        end
+      end
+
+      describe '#cli_path' do
+        it 'returns the default CLI path' do
+          expect(client.cli_path).to eq('/var/vcap/packages/azure-storage-cli/bin/azure-storage-cli')
+        end
+
+        it 'can be overridden by an environment variable' do
+          allow(ENV).to receive(:[]).with('AZURE_STORAGE_CLI_PATH').and_return('/custom/path/to/AzureRM-storage-cli')
+          expect(client.cli_path).to eq('/custom/path/to/AzureRM-storage-cli')
+        end
+      end
+
+      describe  '#exists?' do
+        context 'when the blob exists' do
+          before { allow(client).to receive(:run_cli).with('exists', any_args).and_return([nil, instance_double(Process::Status, exitstatus: 0)]) }
+
+          it('returns true') { expect(client.exists?('some-blob-key')).to be true }
+        end
+
+        context 'when the blob does not exist' do
+          before { allow(client).to receive(:run_cli).with('exists', any_args).and_return([nil, instance_double(Process::Status, exitstatus: 3)]) }
+
+          it('returns false') { expect(client.exists?('some-blob-key')).to be false }
+        end
+      end
+
+      describe '#files_for' do
+        context 'when CLI returns multiple files' do
+          let(:cli_output) { "aa/bb/blob1\ncc/dd/blob2\n" }
+
+          before do
+            allow(client).to receive(:run_cli).
+              with('list', 'some-prefix').
+              and_return([cli_output, instance_double(Process::Status, success?: true)])
+          end
+
+          it 'returns StorageCliBlob instances for each file' do
+            blobs = client.files_for('some-prefix')
+            expect(blobs.map(&:key)).to eq(['aa/bb/blob1', 'cc/dd/blob2'])
+            expect(blobs).to all(be_a(StorageCliBlob))
+          end
+        end
+
+        context 'when CLI returns empty output' do
+          before do
+            allow(client).to receive(:run_cli).
+              with('list', 'some-prefix').
+              and_return(["\n", instance_double(Process::Status, success?: true)])
+          end
+
+          it 'returns an empty array' do
+            expect(client.files_for('some-prefix')).to eq([])
+          end
+        end
+
+        context 'when CLI output has extra whitespace' do
+          let(:cli_output) { "aa/bb/blob1 \n \ncc/dd/blob2\n" }
+
+          before do
+            allow(client).to receive(:run_cli).
+              with('list', 'some-prefix').
+              and_return([cli_output, instance_double(Process::Status, success?: true)])
+          end
+
+          it 'strips and rejects empty lines' do
+            blobs = client.files_for('some-prefix')
+            expect(blobs.map(&:key)).to eq(['aa/bb/blob1', 'cc/dd/blob2'])
+          end
+        end
+      end
+
+      describe '#blob' do
+        let(:properties_json) { '{"etag": "test-etag", "last_modified": "2024-10-01T00:00:00Z", "content_length": 1024}' }
+
+        it 'returns a list of StorageCliBlob instances for a given key' do
+          allow(client).to receive(:run_cli).with('properties', 'bommel/va/li/valid-blob').and_return([properties_json, instance_double(Process::Status, exitstatus: 0)])
+          allow(client).to receive(:run_cli).with('sign', 'bommel/va/li/valid-blob', 'get', '3600s').and_return(['some-url', instance_double(Process::Status, exitstatus: 0)])
+
+          blob = client.blob('valid-blob')
+          expect(blob).to be_a(StorageCliBlob)
+          expect(blob.key).to eq('valid-blob')
+          expect(blob.attributes(:etag, :last_modified, :content_length)).to eq({
+                                                                                  etag: 'test-etag',
+                                                                                  last_modified: '2024-10-01T00:00:00Z',
+                                                                                  content_length: 1024
+                                                                                })
+          expect(blob.internal_download_url).to eq('some-url')
+          expect(blob.public_download_url).to eq('some-url')
+        end
+
+        it 'raises an error if the cli output is empty' do
+          allow(client).to receive(:run_cli).with('properties', 'bommel/no/ne/nonexistent-blob').and_return([nil, instance_double(Process::Status, exitstatus: 0)])
+          expect { client.blob('nonexistent-blob') }.to raise_error(BlobstoreError, /Properties command returned empty output/)
+        end
+
+        it 'raises an error if the cli output is not valid JSON' do
+          allow(client).to receive(:run_cli).with('properties', 'bommel/in/va/invalid-json').and_return(['not a json', instance_double(Process::Status, exitstatus: 0)])
+          expect { client.blob('invalid-json') }.to raise_error(BlobstoreError, /Failed to parse json properties/)
+        end
+      end
+
+      describe '#run_cli' do
+        it 'returns output and status on success' do
+          status = instance_double(Process::Status, success?: true, exitstatus: 0)
+          allow(Open3).to receive(:capture3).with(anything, '-c', anything, 'list', 'arg1').and_return(['ok', '', status])
+
+          output, returned_status = client.send(:run_cli, 'list', 'arg1')
+          expect(output).to eq('ok')
+          expect(returned_status).to eq(status)
+        end
+
+        it 'raises an error on failure' do
+          status = instance_double(Process::Status, success?: false, exitstatus: 1)
+          allow(Open3).to receive(:capture3).with(anything, '-c', anything, 'list', 'arg1').and_return(['', 'error message', status])
+
+          expect do
+            client.send(:run_cli, 'list', 'arg1')
+          end.to raise_error(RuntimeError, /storage-cli list failed with exit code 1/)
+        end
+
+        it 'allows exit code 3 if specified' do
+          status = instance_double(Process::Status, success?: false, exitstatus: 3)
+          allow(Open3).to receive(:capture3).with(anything, '-c', anything, 'list', 'arg1').and_return(['', 'error message', status])
+
+          output, returned_status = client.send(:run_cli, 'list', 'arg1', allow_exit_code_three: true)
+          expect(output).to eq('')
+          expect(returned_status).to eq(status)
+        end
+
+        it 'raises BlobstoreError on Open3 failure' do
+          allow(Open3).to receive(:capture3).and_raise(StandardError.new('Open3 error'))
+
+          expect { client.send(:run_cli, 'list', 'arg1') }.to raise_error(BlobstoreError, /Open3 error/)
+        end
+      end
+    end
+  end
+end

spec/unit/lib/cloud_controller/blobstore/storage_cli/storage_cli_client_spec.rb

@@ -1,5 +1,6 @@
 require 'spec_helper'
 require 'cloud_controller/blobstore/storage_cli/azure_storage_cli_client'
+require 'cloud_controller/blobstore/storage_cli/alioss_storage_cli_client'
 
 module CloudController
   module Blobstore
@@ -28,6 +29,31 @@ module Blobstore
           droplets_cfg.close!
         end
 
+        it 'builds the correct client when JSON has provider aliyun' do
+          droplets_cfg = Tempfile.new(['droplets', '.json'])
+          droplets_cfg.write({
+                               provider:         'aliyun',
+                               access_key_id:     'ali-id',
+                               access_key_secret: 'ali-secret',
+                               endpoint:          'oss-example.aliyuncs.com',
+                               bucket_name:       'ali-bucket'
+                             }.to_json)
+          droplets_cfg.flush
+
+          config_double = instance_double(VCAP::CloudController::Config)
+          allow(VCAP::CloudController::Config).to receive(:config).and_return(config_double)
+          allow(config_double).to receive(:get).with(:storage_cli_config_file_droplets).and_return(droplets_cfg.path)
+
+          client_from_registry = StorageCliClient.build(
+            directory_key: 'dummy-key',
+            root_dir:      'dummy-root',
+            resource_type: 'droplets'
+          )
+          expect(client_from_registry).to be_a(AliStorageCliClient)
+
+          droplets_cfg.close!
+        end
+
         it 'raises an error for an unregistered provider' do
           droplets_cfg = Tempfile.new(['droplets', '.json'])
           droplets_cfg.write(
@@ -195,6 +221,27 @@ def build_client(resource_type)
             end.to raise_error(CloudController::Blobstore::BlobstoreError, /Missing required keys.*#{k}/)
           end
         end
+
+        %w[access_key_id access_key_secret endpoint bucket_name].each do |k|
+          it "raises when #{k} missing for aliyun" do
+            cfg = {
+              'provider'          => 'aliyun',
+              'access_key_id'     => 'ali-id',
+              'access_key_secret' => 'ali-secret',
+              'endpoint'          => 'oss-example.aliyuncs.com',
+              'bucket_name'       => 'ali-bucket'
+            }
+            cfg.delete(k)
+            File.write(droplets_cfg.path, cfg.to_json)
+
+            expect do
+              StorageCliClient.build(directory_key: 'dir', root_dir: 'root', resource_type: 'droplets')
+            end.to raise_error(
+                     CloudController::Blobstore::BlobstoreError,
+                     /Missing required keys.*#{k}/
+                   )
+          end
+        end
       end
 
       describe '#exists? exit code handling' do