Skip to content

Commit bb92b9e

Browse files
authored
bosh-azure-storage-cli based blobstore client (#4476)
* POC: bosh-azure-storage-cli based blobstore client Wrapper client for https://github.com/cloudfoundry/bosh-azure-storage-cli * Introduce general abstraction for storage CLIs
1 parent f95c1d9 commit bb92b9e

14 files changed

+655
-3
lines changed

lib/cloud_controller/blobstore/client_provider.rb

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
require 'cloud_controller/blobstore/client'
22
require 'cloud_controller/blobstore/retryable_client'
33
require 'cloud_controller/blobstore/fog/fog_client'
4-
require 'cloud_controller/blobstore/fog/error_handling_client'
4+
require 'cloud_controller/blobstore/error_handling_client'
55
require 'cloud_controller/blobstore/webdav/dav_client'
66
require 'cloud_controller/blobstore/safe_delete_client'
7+
require 'cloud_controller/blobstore/storage_cli/storage_cli_client'
78
require 'google/apis/errors'
89

910
module CloudController
@@ -12,6 +13,9 @@ class ClientProvider
1213
def self.provide(options:, directory_key:, root_dir: nil, resource_type: nil)
1314
if options[:blobstore_type].blank? || (options[:blobstore_type] == 'fog')
1415
provide_fog(options, directory_key, root_dir)
16+
elsif options[:blobstore_type] == 'storage-cli'
17+
# storage-cli is an experimental feature and not yet fully implemented. !!! DO NOT USE IN PRODUCTION !!!
18+
provide_storage_cli(options, directory_key, root_dir)
1519
else
1620
provide_webdav(options, directory_key, root_dir)
1721
end
@@ -65,6 +69,22 @@ def provide_webdav(options, directory_key, root_dir)
6569

6670
Client.new(SafeDeleteClient.new(retryable_client, root_dir))
6771
end
72+
73+
def provide_storage_cli(options, directory_key, root_dir)
74+
raise BlobstoreError.new('connection_config for storage-cli is not provided') unless options[:connection_config]
75+
76+
client = StorageCliClient.build(connection_config: options.fetch(:connection_config),
77+
directory_key: directory_key,
78+
root_dir: root_dir,
79+
min_size: options[:minimum_size],
80+
max_size: options[:maximum_size])
81+
82+
logger = Steno.logger('cc.blobstore.storage_cli_client')
83+
errors = [StandardError]
84+
retryable_client = RetryableClient.new(client:, errors:, logger:)
85+
86+
Client.new(SafeDeleteClient.new(retryable_client, root_dir))
87+
end
6888
end
6989
end
7090
end
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
module CloudController
2+
module Blobstore
3+
class AzureStorageCliClient < StorageCliClient
4+
def cli_path
5+
ENV['AZURE_STORAGE_CLI_PATH'] || '/var/vcap/packages/azure-storage-cli/bin/azure-storage-cli'
6+
end
7+
8+
def build_config(connection_config)
9+
{
10+
account_name: connection_config[:azure_storage_account_name],
11+
account_key: connection_config[:azure_storage_access_key],
12+
container_name: @directory_key,
13+
environment: connection_config[:environment]
14+
}.compact
15+
end
16+
17+
CloudController::Blobstore::StorageCliClient.register('AzureRM', AzureStorageCliClient)
18+
end
19+
end
20+
end
Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
module CloudController
2+
module Blobstore
3+
class StorageCliBlob < Blob
4+
attr_reader :key
5+
6+
def initialize(key, properties: nil, signed_url: nil)
7+
@key = key
8+
@signed_url = signed_url if signed_url
9+
# Set properties to an empty hash if nil to avoid nil errors
10+
@properties = properties || {}
11+
end
12+
13+
def internal_download_url
14+
signed_url
15+
end
16+
17+
def public_download_url
18+
signed_url
19+
end
20+
21+
def attributes(*keys)
22+
@attributes ||= {
23+
etag: @properties.fetch('etag', nil),
24+
last_modified: @properties.fetch('last_modified', nil),
25+
content_length: @properties.fetch('content_length', nil),
26+
created_at: @properties.fetch('created_at', nil)
27+
}
28+
29+
return @attributes if keys.empty?
30+
31+
@attributes.select { |key, _| keys.include? key }
32+
end
33+
34+
private
35+
36+
def signed_url
37+
raise BlobstoreError.new('StorageCliBlob not configured with a signed URL') unless @signed_url
38+
39+
@signed_url
40+
end
41+
end
42+
end
43+
end
Lines changed: 221 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
require 'open3'
2+
require 'tempfile'
3+
require 'tmpdir'
4+
require 'fileutils'
5+
require 'cloud_controller/blobstore/base_client'
6+
require 'cloud_controller/blobstore/storage_cli/storage_cli_blob'
7+
8+
module CloudController
9+
module Blobstore
10+
class StorageCliClient < BaseClient
11+
attr_reader :root_dir, :min_size, :max_size
12+
13+
@registry = {}
14+
15+
class << self
16+
attr_reader :registry
17+
18+
def register(provider, klass)
19+
registry[provider] = klass
20+
end
21+
22+
def build(connection_config:, directory_key:, root_dir:, min_size: nil, max_size: nil)
23+
provider = connection_config[:provider]
24+
raise 'Missing connection_config[:provider]' if provider.nil?
25+
26+
impl_class = registry[provider]
27+
raise "No storage CLI client registered for provider #{provider}" unless impl_class
28+
29+
impl_class.new(connection_config:, directory_key:, root_dir:, min_size:, max_size:)
30+
end
31+
end
32+
33+
def initialize(connection_config:, directory_key:, root_dir:, min_size: nil, max_size: nil)
34+
@cli_path = cli_path
35+
@directory_key = directory_key
36+
@root_dir = root_dir
37+
@min_size = min_size || 0
38+
@max_size = max_size
39+
config = build_config(connection_config)
40+
@config_file = write_config_file(config)
41+
@fork = connection_config.fetch(:fork, false)
42+
end
43+
44+
def local?
45+
false
46+
end
47+
48+
def exists?(blobstore_key)
49+
key = partitioned_key(blobstore_key)
50+
_, status = run_cli('exists', key, allow_exit_code_three: true)
51+
52+
if status.exitstatus == 0
53+
return true
54+
elsif status.exitstatus == 3
55+
return false
56+
end
57+
58+
false
59+
end
60+
61+
def download_from_blobstore(source_key, destination_path, mode: nil)
62+
FileUtils.mkdir_p(File.dirname(destination_path))
63+
run_cli('get', partitioned_key(source_key), destination_path)
64+
65+
File.chmod(mode, destination_path) if mode
66+
end
67+
68+
def cp_to_blobstore(source_path, destination_key)
69+
start = Time.now.utc
70+
log_entry = 'cp-skip'
71+
size = -1
72+
73+
logger.info('cp-start', destination_key: destination_key, source_path: source_path, bucket: @directory_key)
74+
75+
File.open(source_path) do |file|
76+
size = file.size
77+
next unless within_limits?(size)
78+
79+
run_cli('put', source_path, partitioned_key(destination_key))
80+
log_entry = 'cp-finish'
81+
end
82+
83+
duration = Time.now.utc - start
84+
logger.info(log_entry,
85+
destination_key: destination_key,
86+
duration_seconds: duration,
87+
size: size)
88+
end
89+
90+
def cp_file_between_keys(source_key, destination_key)
91+
if @fork
92+
run_cli('copy', partitioned_key(source_key), partitioned_key(destination_key))
93+
else
94+
# Azure CLI doesn't support server-side copy yet, so fallback to local copy
95+
Tempfile.create('blob-copy') do |tmp|
96+
download_from_blobstore(source_key, tmp.path)
97+
cp_to_blobstore(tmp.path, destination_key)
98+
end
99+
end
100+
end
101+
102+
def delete_all(_=nil)
103+
# page_size is currently not considered. Azure SDK / API has a limit of 5000
104+
pass unless @fork
105+
106+
# Currently, storage-cli does not support bulk deletion.
107+
run_cli('delete-recursive', @root_dir)
108+
end
109+
110+
def delete_all_in_path(path)
111+
pass unless @fork
112+
113+
# Currently, storage-cli does not support bulk deletion.
114+
run_cli('delete-recursive', partitioned_key(path))
115+
end
116+
117+
def delete(key)
118+
run_cli('delete', partitioned_key(key))
119+
end
120+
121+
def delete_blob(blob)
122+
delete(blob.key)
123+
end
124+
125+
def blob(key)
126+
if @fork
127+
properties = properties(key)
128+
return nil if properties.nil? || properties.empty?
129+
130+
signed_url = sign_url(partitioned_key(key), verb: 'get', expires_in_seconds: 3600)
131+
StorageCliBlob.new(key, properties:, signed_url:)
132+
elsif exists?(key)
133+
# Azure CLI does not support getting blob properties directly, so fallback to local check
134+
signed_url = sign_url(partitioned_key(key), verb: 'get', expires_in_seconds: 3600)
135+
StorageCliBlob.new(key, signed_url:)
136+
end
137+
end
138+
139+
def files_for(prefix, _ignored_directory_prefixes=[])
140+
return nil unless @fork
141+
142+
files, _status = run_cli('list', prefix)
143+
files.split("\n").map(&:strip).reject(&:empty?).map { |file| StorageCliBlob.new(file) }
144+
end
145+
146+
def ensure_bucket_exists
147+
return unless @fork
148+
149+
run_cli('ensure-bucket-exists')
150+
end
151+
152+
private
153+
154+
def run_cli(command, *args, allow_exit_code_three: false)
155+
logger.info("[storage_cli_client] Running storage-cli: #{@cli_path} -c #{@config_file} #{command} #{args.join(' ')}")
156+
157+
begin
158+
stdout, stderr, status = Open3.capture3(@cli_path, '-c', @config_file, command, *args)
159+
rescue StandardError => e
160+
raise BlobstoreError.new(e.inspect)
161+
end
162+
163+
unless status.success? || (allow_exit_code_three && status.exitstatus == 3)
164+
raise "storage-cli #{command} failed with exit code #{status.exitstatus}, output: '#{stdout}', error: '#{stderr}'"
165+
end
166+
167+
[stdout, status]
168+
end
169+
170+
def sign_url(key, verb:, expires_in_seconds:)
171+
stdout, _status = run_cli('sign', key, verb.to_s.downcase, "#{expires_in_seconds}s")
172+
stdout.strip
173+
end
174+
175+
def properties(key)
176+
stdout, _status = run_cli('properties', partitioned_key(key))
177+
# stdout is expected to be in JSON format - raise an error if it is nil, empty or something unexpected
178+
raise BlobstoreError.new("Properties command returned empty output for key: #{key}") if stdout.nil? || stdout.empty?
179+
180+
begin
181+
properties = Oj.load(stdout)
182+
rescue StandardError => e
183+
raise BlobstoreError.new("Failed to parse json properties for key: #{key}, error: #{e.message}")
184+
end
185+
186+
properties
187+
end
188+
189+
def cli_path
190+
raise NotImplementedError
191+
end
192+
193+
def build_config(connection_config)
194+
raise NotImplementedError
195+
end
196+
197+
def write_config_file(config)
198+
# TODO: Consider to move the config generation into capi-release
199+
config_dir = File.join(tmpdir, 'blobstore-configs')
200+
FileUtils.mkdir_p(config_dir)
201+
202+
config_file_path = File.join(config_dir, "#{@directory_key}.json")
203+
File.open(config_file_path, 'w', 0o600) do |f|
204+
f.write(Oj.dump(config.transform_keys(&:to_s)))
205+
end
206+
config_file_path
207+
end
208+
209+
def tmpdir
210+
VCAP::CloudController::Config.config.get(:directories, :tmpdir)
211+
rescue StandardError
212+
# Fallback to a temporary directory if the config is not set (e.g. for cc-deployment-updater
213+
Dir.mktmpdir('cc_blobstore')
214+
end
215+
216+
def logger
217+
@logger ||= Steno.logger('cc.blobstore.storage_cli_client')
218+
end
219+
end
220+
end
221+
end

lib/cloud_controller/config_schemas/api_schema.rb

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,13 +198,15 @@ class ApiSchema < VCAP::Config
198198
minimum_size: Integer,
199199
resource_directory_key: String,
200200
fog_connection: Hash,
201+
optional(:connection_config) => Hash,
201202
fog_aws_storage_options: Hash,
202203
fog_gcp_storage_options: Hash
203204
},
204205

205206
buildpacks: {
206207
buildpack_directory_key: String,
207208
fog_connection: Hash,
209+
optional(:connection_config) => Hash,
208210
fog_aws_storage_options: Hash,
209211
fog_gcp_storage_options: Hash
210212
},
@@ -214,6 +216,7 @@ class ApiSchema < VCAP::Config
214216
max_valid_packages_stored: Integer,
215217
app_package_directory_key: String,
216218
fog_connection: Hash,
219+
optional(:connection_config) => Hash,
217220
fog_aws_storage_options: Hash,
218221
fog_gcp_storage_options: Hash
219222
},
@@ -222,6 +225,7 @@ class ApiSchema < VCAP::Config
222225
droplet_directory_key: String,
223226
max_staged_droplets_stored: Integer,
224227
fog_connection: Hash,
228+
optional(:connection_config) => Hash,
225229
fog_aws_storage_options: Hash,
226230
fog_gcp_storage_options: Hash
227231
},

lib/cloud_controller/config_schemas/clock_schema.rb

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,13 +109,15 @@ class ClockSchema < VCAP::Config
109109
minimum_size: Integer,
110110
resource_directory_key: String,
111111
fog_connection: Hash,
112+
optional(:connection_config) => Hash,
112113
fog_aws_storage_options: Hash,
113114
fog_gcp_storage_options: Hash
114115
},
115116

116117
buildpacks: {
117118
buildpack_directory_key: String,
118119
fog_connection: Hash,
120+
optional(:connection_config) => Hash,
119121
fog_aws_storage_options: Hash,
120122
fog_gcp_storage_options: Hash
121123
},
@@ -124,13 +126,15 @@ class ClockSchema < VCAP::Config
124126
max_package_size: Integer,
125127
app_package_directory_key: String,
126128
fog_connection: Hash,
129+
optional(:connection_config) => Hash,
127130
fog_aws_storage_options: Hash,
128131
fog_gcp_storage_options: Hash
129132
},
130133

131134
droplets: {
132135
droplet_directory_key: String,
133136
fog_connection: Hash,
137+
optional(:connection_config) => Hash,
134138
fog_aws_storage_options: Hash,
135139
fog_gcp_storage_options: Hash
136140
},

0 commit comments

Comments
 (0)