Skip to content

Commit f8195dd

Browse files
johhajohha
committed
rename config
1 parent 0b8c86a commit f8195dd

File tree

10 files changed

+60
-35
lines changed

10 files changed

+60
-35
lines changed

‎lib/cloud_controller/blobstore/client_provider.rb

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,8 @@ def self.provide(options:, directory_key:, root_dir: nil, resource_type: nil)
1414
if options[:blobstore_type].blank? || (options[:blobstore_type] == 'fog')
1515
provide_fog(options, directory_key, root_dir)
1616
elsif options[:blobstore_type] == 'storage-cli'
17+
# storage-cli is an experimental feature and not yet fully implemented. !!! DO NOT USE IN PRODUCTION !!!
1718
provide_storage_cli(options, directory_key, root_dir)
18-
elsif options[:blobstore_type] == 'storage-cli-fork'
19-
# Just for testing not yet merged features in bosh-azure-cli
20-
provide_storage_cli(options, directory_key, root_dir, fork: true)
2119
else
2220
provide_webdav(options, directory_key, root_dir)
2321
end
@@ -72,13 +70,14 @@ def provide_webdav(options, directory_key, root_dir)
7270
Client.new(SafeDeleteClient.new(retryable_client, root_dir))
7371
end
7472

75-
def provide_storage_cli(options, directory_key, root_dir, fork: false)
76-
client = StorageCliClient.build(fog_connection: options.fetch(:fog_connection),
73+
def provide_storage_cli(options, directory_key, root_dir)
74+
raise BlobstoreError.new('connection_config for storage-cli is not provided') unless options[:connection_config]
75+
76+
client = StorageCliClient.build(connection_config: options.fetch(:connection_config),
7777
directory_key: directory_key,
7878
root_dir: root_dir,
7979
min_size: options[:minimum_size],
80-
max_size: options[:maximum_size],
81-
fork: fork)
80+
max_size: options[:maximum_size])
8281

8382
logger = Steno.logger('cc.blobstore.storage_cli_client')
8483
errors = [StandardError]

‎lib/cloud_controller/blobstore/storage_cli/azure_storage_cli_client.rb

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,20 @@
11
module CloudController
22
module Blobstore
33
class AzureStorageCliClient < StorageCliClient
4-
StorageCliClient.register('AzureRM', CloudController::Blobstore::AzureStorageCliClient)
5-
64
def cli_path
75
ENV['AZURE_STORAGE_CLI_PATH'] || '/var/vcap/packages/azure-storage-cli/bin/azure-storage-cli'
86
end
97

10-
def build_config(fog_connection)
8+
def build_config(connection_config)
119
{
12-
account_name: fog_connection[:azure_storage_account_name],
13-
account_key: fog_connection[:azure_storage_access_key],
10+
account_name: connection_config[:azure_storage_account_name],
11+
account_key: connection_config[:azure_storage_access_key],
1412
container_name: @directory_key,
15-
environment: fog_connection[:environment]
13+
environment: connection_config[:environment]
1614
}.compact
1715
end
16+
17+
CloudController::Blobstore::StorageCliClient.register('AzureRM', AzureStorageCliClient)
1818
end
1919
end
2020
end

‎lib/cloud_controller/blobstore/storage_cli/storage_cli_client.rb

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
require 'open3'
22
require 'tempfile'
3+
require 'tmpdir'
34
require 'fileutils'
45
require 'cloud_controller/blobstore/base_client'
56
require 'cloud_controller/blobstore/storage_cli/storage_cli_blob'
@@ -16,27 +17,26 @@ def register(provider, klass)
1617
@registry[provider] = klass
1718
end
1819

19-
def build(fog_connection:, directory_key:, root_dir:, min_size: nil, max_size: nil, fork: false)
20-
provider = fog_connection[:provider]
21-
raise 'Missing fog_connection[:provider]' if provider.nil?
20+
def build(connection_config:, directory_key:, root_dir:, min_size: nil, max_size: nil)
21+
provider = connection_config[:provider]
22+
raise 'Missing connection_config[:provider]' if provider.nil?
2223

2324
impl_class = @registry[provider]
2425
raise "No storage CLI client registered for provider #{provider}" unless impl_class
2526

26-
impl_class.new(fog_connection:, directory_key:, root_dir:, min_size:, max_size:, fork:)
27+
impl_class.new(connection_config:, directory_key:, root_dir:, min_size:, max_size:, fork:)
2728
end
2829
end
2930

30-
def initialize(fog_connection:, directory_key:, root_dir:, min_size: nil, max_size: nil, fork: false)
31+
def initialize(connection_config:, directory_key:, root_dir:, min_size: nil, max_size: nil)
3132
@cli_path = cli_path
3233
@directory_key = directory_key
3334
@root_dir = root_dir
3435
@min_size = min_size || 0
3536
@max_size = max_size
36-
@fork = fork
37-
38-
config = build_config(fog_connection)
37+
config = build_config(connection_config)
3938
@config_file = write_config_file(config)
39+
@fork = connection_config.fetch(:fork, false)
4040
end
4141

4242
def local?
@@ -142,9 +142,9 @@ def files_for(prefix, _ignored_directory_prefixes=[])
142142
end
143143

144144
def ensure_bucket_exists
145-
pass unless @fork
145+
return unless @fork
146146

147-
run_cli('ensure-bucket-exists', @directory_key)
147+
run_cli('ensure-bucket-exists')
148148
end
149149

150150
private
@@ -188,23 +188,27 @@ def cli_path
188188
raise NotImplementedError
189189
end
190190

191-
def build_config(fog_connection)
191+
def build_config(connection_config)
192192
raise NotImplementedError
193193
end
194194

195195
def write_config_file(config)
196+
# TODO: Consider to move the config generation into capi-release
196197
config_dir = File.join(tmpdir, 'blobstore-configs')
197198
FileUtils.mkdir_p(config_dir)
198199

199200
config_file_path = File.join(config_dir, "#{@directory_key}.json")
200201
File.open(config_file_path, 'w', 0o600) do |f|
201-
f.write(Oj.dump(config))
202+
f.write(Oj.dump(config.transform_keys(&:to_s)))
202203
end
203204
config_file_path
204205
end
205206

206207
def tmpdir
207208
VCAP::CloudController::Config.config.get(:directories, :tmpdir)
209+
rescue StandardError
210+
# Fallback to a temporary directory if the config is not set (e.g. for cc-deployment-updater
211+
Dir.mktmpdir('cc_blobstore')
208212
end
209213

210214
def logger

‎lib/cloud_controller/config_schemas/base/api_schema.rb

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,13 +170,15 @@ class ApiSchema < VCAP::Config
170170
minimum_size: Integer,
171171
resource_directory_key: String,
172172
fog_connection: Hash,
173+
optional(:connection_config) => Hash,
173174
fog_aws_storage_options: Hash,
174175
fog_gcp_storage_options: Hash
175176
},
176177

177178
buildpacks: {
178179
buildpack_directory_key: String,
179180
fog_connection: Hash,
181+
optional(:connection_config) => Hash,
180182
fog_aws_storage_options: Hash,
181183
fog_gcp_storage_options: Hash
182184
},
@@ -186,6 +188,7 @@ class ApiSchema < VCAP::Config
186188
max_valid_packages_stored: Integer,
187189
app_package_directory_key: String,
188190
fog_connection: Hash,
191+
optional(:connection_config) => Hash,
189192
fog_aws_storage_options: Hash,
190193
fog_gcp_storage_options: Hash
191194
},
@@ -194,6 +197,7 @@ class ApiSchema < VCAP::Config
194197
droplet_directory_key: String,
195198
max_staged_droplets_stored: Integer,
196199
fog_connection: Hash,
200+
optional(:connection_config) => Hash,
197201
fog_aws_storage_options: Hash,
198202
fog_gcp_storage_options: Hash
199203
},

‎lib/cloud_controller/config_schemas/base/clock_schema.rb

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,13 +83,15 @@ class ClockSchema < VCAP::Config
8383
minimum_size: Integer,
8484
resource_directory_key: String,
8585
fog_connection: Hash,
86+
optional(:connection_config) => Hash,
8687
fog_aws_storage_options: Hash,
8788
fog_gcp_storage_options: Hash
8889
},
8990

9091
buildpacks: {
9192
buildpack_directory_key: String,
9293
fog_connection: Hash,
94+
optional(:connection_config) => Hash,
9395
fog_aws_storage_options: Hash,
9496
fog_gcp_storage_options: Hash
9597
},
@@ -98,13 +100,15 @@ class ClockSchema < VCAP::Config
98100
max_package_size: Integer,
99101
app_package_directory_key: String,
100102
fog_connection: Hash,
103+
optional(:connection_config) => Hash,
101104
fog_aws_storage_options: Hash,
102105
fog_gcp_storage_options: Hash
103106
},
104107

105108
droplets: {
106109
droplet_directory_key: String,
107110
fog_connection: Hash,
111+
optional(:connection_config) => Hash,
108112
fog_aws_storage_options: Hash,
109113
fog_gcp_storage_options: Hash
110114
},

‎lib/cloud_controller/config_schemas/base/deployment_updater_schema.rb

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,13 +83,15 @@ class DeploymentUpdaterSchema < VCAP::Config
8383
minimum_size: Integer,
8484
resource_directory_key: String,
8585
fog_connection: Hash,
86+
optional(:connection_config) => Hash,
8687
fog_aws_storage_options: Hash,
8788
fog_gcp_storage_options: Hash
8889
},
8990

9091
buildpacks: {
9192
buildpack_directory_key: String,
9293
fog_connection: Hash,
94+
optional(:connection_config) => Hash,
9395
fog_aws_storage_options: Hash,
9496
fog_gcp_storage_options: Hash
9597
},
@@ -98,13 +100,15 @@ class DeploymentUpdaterSchema < VCAP::Config
98100
max_package_size: Integer,
99101
app_package_directory_key: String,
100102
fog_connection: Hash,
103+
optional(:connection_config) => Hash,
101104
fog_aws_storage_options: Hash,
102105
fog_gcp_storage_options: Hash
103106
},
104107

105108
droplets: {
106109
droplet_directory_key: String,
107110
fog_connection: Hash,
111+
optional(:connection_config) => Hash,
108112
fog_aws_storage_options: Hash,
109113
fog_gcp_storage_options: Hash
110114
},

‎lib/cloud_controller/config_schemas/base/worker_schema.rb

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,13 +74,15 @@ class WorkerSchema < VCAP::Config
7474
minimum_size: Integer,
7575
resource_directory_key: String,
7676
fog_connection: Hash,
77+
optional(:connection_config) => Hash,
7778
fog_aws_storage_options: Hash,
7879
fog_gcp_storage_options: Hash
7980
},
8081

8182
buildpacks: {
8283
buildpack_directory_key: String,
8384
fog_connection: Hash,
85+
optional(:connection_config) => Hash,
8486
fog_aws_storage_options: Hash,
8587
fog_gcp_storage_options: Hash
8688
},
@@ -90,13 +92,15 @@ class WorkerSchema < VCAP::Config
9092
max_valid_packages_stored: Integer,
9193
app_package_directory_key: String,
9294
fog_connection: Hash,
95+
optional(:connection_config) => Hash,
9396
fog_aws_storage_options: Hash,
9497
fog_gcp_storage_options: Hash
9598
},
9699

97100
droplets: {
98101
droplet_directory_key: String,
99102
fog_connection: Hash,
103+
optional(:connection_config) => Hash,
100104
fog_aws_storage_options: Hash,
101105
fog_gcp_storage_options: Hash
102106
},

‎spec/unit/lib/cloud_controller/blobstore/client_provider_spec.rb

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,13 +133,18 @@ module Blobstore
133133
let(:storage_cli_client_mock) { class_double(CloudController::Blobstore::StorageCliClient) }
134134

135135
before do
136-
options.merge!(fog_connection: {}, minimum_size: 100, maximum_size: 1000)
136+
options.merge!(connection_config: {}, minimum_size: 100, maximum_size: 1000)
137137
end
138138

139139
it 'provides a storage-cli client' do
140140
allow(StorageCliClient).to receive(:build).and_return(storage_cli_client_mock)
141141
ClientProvider.provide(options:, directory_key:, root_dir:)
142-
expect(StorageCliClient).to have_received(:build).with(fog_connection: {}, directory_key: directory_key, root_dir: root_dir, min_size: 100, max_size: 1000, fork: false)
142+
expect(StorageCliClient).to have_received(:build).with(connection_config: {}, directory_key: directory_key, root_dir: root_dir, min_size: 100, max_size: 1000)
143+
end
144+
145+
it 'raises an error if connection_config is not provided' do
146+
options.delete(:connection_config)
147+
expect { ClientProvider.provide(options:, directory_key:, root_dir:) }.to raise_error(BlobstoreError, 'connection_config for storage-cli is not provided')
143148
end
144149
end
145150
end

‎spec/unit/lib/cloud_controller/blobstore/storage_cli/azure_storage_cli_client_spec.rb

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,16 @@
66
module CloudController
77
module Blobstore
88
RSpec.describe AzureStorageCliClient do
9-
subject(:client) { AzureStorageCliClient.new(fog_connection: fog_connection, directory_key: directory_key, root_dir: 'bommel', fork: true) }
9+
subject(:client) { AzureStorageCliClient.new(connection_config: connection_config, directory_key: directory_key, root_dir: 'bommel') }
1010
let(:directory_key) { 'my-bucket' }
11-
let(:fog_connection) do
11+
let(:connection_config) do
1212
{
1313
azure_storage_access_key: 'some-access-key',
1414
azure_storage_account_name: 'some-account-name',
1515
container_name: directory_key,
1616
environment: 'AzureCloud',
17-
provider: 'AzureRM'
17+
provider: 'AzureRM',
18+
fork: true
1819
}
1920
end
2021
let(:downloaded_file) do
@@ -39,7 +40,7 @@ module Blobstore
3940
allow(client).to receive(:run_cli).with('delete', anything).and_return([nil, instance_double(Process::Status, exitstatus: 0)])
4041
allow(client).to receive(:run_cli).with('delete-recursive', anything).and_return([nil, instance_double(Process::Status, exitstatus: 0)])
4142
allow(client).to receive(:run_cli).with('list', anything).and_return(["aa/bb/blob1\ncc/dd/blob2\n", instance_double(Process::Status, exitstatus: 0)])
42-
allow(client).to receive(:run_cli).with('ensure-bucket-exists', anything).and_return([nil, instance_double(Process::Status, exitstatus: 0)])
43+
allow(client).to receive(:run_cli).with('ensure-bucket-exists').and_return([nil, instance_double(Process::Status, exitstatus: 0)])
4344
allow(client).to receive(:run_cli).with('properties', anything).and_return(['{"dummy": "json"}', instance_double(Process::Status, exitstatus: 0)])
4445
allow(client).to receive(:run_cli).with('sign', anything, 'get', '3600s').and_return(['some-url', instance_double(Process::Status, exitstatus: 0)])
4546
end

‎spec/unit/lib/cloud_controller/blobstore/storage_cli/storage_cli_client_spec.rb

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,20 +12,20 @@ def initialize(*); end
1212
before { StorageCliClient.register('DummyProvider', DummyClient) }
1313

1414
it 'builds the correct client' do
15-
client_from_registry = StorageCliClient.build(fog_connection: { provider: 'DummyProvider' }, directory_key: 'dummy-key', root_dir: 'dummy-root')
15+
client_from_registry = StorageCliClient.build(connection_config: { provider: 'DummyProvider' }, directory_key: 'dummy-key', root_dir: 'dummy-root')
1616
expect(client_from_registry).to be_a(DummyClient)
1717
end
1818

1919
it 'raises an error for an unregistered provider' do
2020
expect do
21-
StorageCliClient.build(fog_connection: { provider: 'UnknownProvider' }, directory_key: 'dummy-key', root_dir: 'dummy-root')
21+
StorageCliClient.build(connection_config: { provider: 'UnknownProvider' }, directory_key: 'dummy-key', root_dir: 'dummy-root')
2222
end.to raise_error(RuntimeError, 'No storage CLI client registered for provider UnknownProvider')
2323
end
2424

2525
it 'raises an error if provider is missing' do
2626
expect do
27-
StorageCliClient.build(fog_connection: {}, directory_key: 'dummy-key', root_dir: 'dummy-root')
28-
end.to raise_error(RuntimeError, 'Missing fog_connection[:provider]')
27+
StorageCliClient.build(connection_config: {}, directory_key: 'dummy-key', root_dir: 'dummy-root')
28+
end.to raise_error(RuntimeError, 'Missing connection_config[:provider]')
2929
end
3030
end
3131
end

0 commit comments

Comments
 (0)