Merge pull request #2690 from cloudfoundry/azure-bump-rebase

Azure bump rebase

Author: sweinstein22

Gemfile

@@ -53,14 +53,17 @@ gem 'activemodel', '~> 6.1.4'
 gem 'railties', '~> 6.1.4'
 
 # Blobstore and Bits Service Dependencies
-gem 'azure-storage', '0.14.0.preview' # https://github.com/Azure/azure-storage-ruby/issues/122
 gem 'bits_service_client', '~> 3.3', '>= 3.3.0'
+
+gem 'azure-storage-blob', git: 'https://github.com/sethboyles/azure-storage-ruby.git', branch: 'x-ms-blob-content-type-fix-1.1'
+
 gem 'fog-aliyun'
 gem 'fog-aws'
 gem 'fog-azure-rm', git: 'https://github.com/fog/fog-azure-rm.git', branch: 'fog-arm-cf'
-gem 'fog-google'
+gem 'fog-google', '~> 1.18.0'
 gem 'fog-local'
 gem 'fog-openstack'
+gem 'fog-core', '~> 2.1.2'
 
 gem 'cf-uaa-lib', '~> 4.0.1'
 gem 'vcap-concurrency', git: 'https://github.com/cloudfoundry/vcap-concurrency.git', ref: '2a5b0179'

Gemfile.lock

@@ -7,11 +7,12 @@ GIT
 
 GIT
   remote: https://github.com/fog/fog-azure-rm.git
-  revision: 79a738331538c6bdba806774cb11af8977979f8e
+  revision: 69ba1d7e9f82b7a71214b92fa6fa95169b539ed3
   branch: fog-arm-cf
   specs:
     fog-azure-rm (0.4.7)
-      azure-storage (>= 0.11.5.preview, < 1.0)
+      azure-storage-blob (~> 1.1)
+      azure-storage-common (~> 1.1)
       azure_mgmt_compute (~> 0.9.0)
       azure_mgmt_dns (~> 0.9.0)
       azure_mgmt_key_vault (~> 0.9.0)
@@ -20,11 +21,24 @@ GIT
       azure_mgmt_sql (~> 0.9.0)
       azure_mgmt_storage (~> 0.9.0)
       azure_mgmt_traffic_manager (~> 0.9.0)
-      fog-core (~> 1.43)
-      fog-json (~> 1.0.2)
+      fog-core (>= 1.43)
+      fog-json (>= 1.0.2)
       mime-types (~> 3.0)
       vhd (= 0.0.4)
 
+GIT
+  remote: https://github.com/sethboyles/azure-storage-ruby.git
+  revision: d4737cdfcef3f3c4c2463a54690b6b73c0f6f707
+  branch: x-ms-blob-content-type-fix-1.1
+  specs:
+    azure-storage-blob (1.1.0)
+      azure-core (~> 0.1.13)
+      azure-storage-common (~> 1.0)
+      nokogiri (~> 1.6, >= 1.6.8)
+    azure-storage-common (1.1.0)
+      azure-core (~> 0.1.13)
+      nokogiri (~> 1.6, >= 1.6.8)
+
 GIT
   remote: https://github.com/sporkrb/spork
   revision: 224df492657e617a0c93c0319e78f0eefee5b636
@@ -66,15 +80,10 @@ GEM
       i18n
     ast (2.4.2)
     awesome_print (1.9.2)
-    azure-core (0.1.14)
+    azure-core (0.1.15)
       faraday (~> 0.9)
       faraday_middleware (~> 0.10)
       nokogiri (~> 1.6)
-    azure-storage (0.14.0.preview)
-      azure-core (~> 0.1)
-      faraday (~> 0.9)
-      faraday_middleware (~> 0.10)
-      nokogiri (~> 1.6, >= 1.6.8)
     azure_mgmt_compute (0.9.0)
       ms_rest_azure (~> 0.7.0)
     azure_mgmt_dns (0.9.0)
@@ -94,7 +103,7 @@ GEM
     backport (1.2.0)
     beefcake (1.0.0)
     benchmark (0.2.0)
-    bit-struct (0.16)
+    bit-struct (0.17)
     bits_service_client (3.4.0)
       activesupport
       statsd-ruby (~> 1.4.0)
@@ -123,8 +132,7 @@ GEM
     debase (0.2.5.beta2)
       debase-ruby_core_source (>= 0.10.12)
     debase-ruby_core_source (0.10.12)
-    declarative (0.0.10)
-    declarative-option (0.1.0)
+    declarative (0.0.20)
     delayed_job (4.1.9)
       activesupport (>= 3.0, < 6.2)
     diff-lcs (1.5.0)
@@ -142,13 +150,13 @@ GEM
       eventmachine (>= 1.0.0.beta.4)
     erubi (1.10.0)
     eventmachine (1.2.7)
-    excon (0.90.0)
-    faraday (0.15.4)
+    excon (0.91.0)
+    faraday (0.17.4)
       multipart-post (>= 1.2, < 3)
-    faraday-cookie_jar (0.0.6)
-      faraday (>= 0.7.4)
+    faraday-cookie_jar (0.0.7)
+      faraday (>= 0.8.0)
       http-cookie (~> 1.0.0)
-    faraday_middleware (0.12.2)
+    faraday_middleware (0.14.0)
       faraday (>= 0.7.4, < 1.0)
     ffi (1.15.5)
     ffi-compiler (1.0.1)
@@ -162,41 +170,66 @@ GEM
       fog-json
       ipaddress (~> 0.8)
       xml-simple (~> 1.1)
-    fog-aws (2.0.1)
-      fog-core (~> 1.38)
-      fog-json (~> 1.0)
+    fog-aws (3.12.0)
+      fog-core (~> 2.1)
+      fog-json (~> 1.1)
       fog-xml (~> 0.1)
       ipaddress (~> 0.8)
-    fog-core (1.45.0)
+    fog-core (2.1.2)
       builder
       excon (~> 0.58)
       formatador (~> 0.2)
-    fog-google (1.7.1)
+      mime-types
+    fog-google (1.18.0)
+      fog-core (< 2.3)
+      fog-json (~> 1.2)
+      fog-xml (~> 0.1.0)
+      google-apis-compute_v1 (~> 0.14)
+      google-apis-dns_v1 (~> 0.12)
+      google-apis-iamcredentials_v1 (~> 0.6)
+      google-apis-monitoring_v3 (~> 0.12)
+      google-apis-pubsub_v1 (~> 0.7)
+      google-apis-sqladmin_v1beta4 (~> 0.13)
+      google-apis-storage_v1 (~> 0.6)
+      google-cloud-env (~> 1.2)
+    fog-json (1.2.0)
       fog-core
-      fog-json
-      fog-xml
-      google-api-client (~> 0.23.0)
-    fog-json (1.0.2)
-      fog-core (~> 1.0)
       multi_json (~> 1.10)
     fog-local (0.8.0)
       fog-core (>= 1.27, < 3.0)
-    fog-openstack (0.3.10)
-      fog-core (>= 1.45, <= 2.1.0)
+    fog-openstack (1.0.11)
+      fog-core (~> 2.1)
       fog-json (>= 1.0)
       ipaddress (>= 0.8)
-    fog-xml (0.1.3)
+    fog-xml (0.1.4)
       fog-core
       nokogiri (>= 1.5.11, < 2.0.0)
     formatador (0.3.0)
-    google-api-client (0.23.9)
+    google-apis-compute_v1 (0.25.0)
+      google-apis-core (>= 0.4, < 2.a)
+    google-apis-core (0.4.2)
       addressable (~> 2.5, >= 2.5.1)
-      googleauth (>= 0.5, < 0.7.0)
-      httpclient (>= 2.8.1, < 3.0)
-      mime-types (~> 3.0)
+      googleauth (>= 0.16.2, < 2.a)
+      httpclient (>= 2.8.1, < 3.a)
+      mini_mime (~> 1.0)
       representable (~> 3.0)
-      retriable (>= 2.0, < 4.0)
-      signet (~> 0.9)
+      retriable (>= 2.0, < 4.a)
+      rexml
+      webrick
+    google-apis-dns_v1 (0.19.0)
+      google-apis-core (>= 0.4, < 2.a)
+    google-apis-iamcredentials_v1 (0.10.0)
+      google-apis-core (>= 0.4, < 2.a)
+    google-apis-monitoring_v3 (0.21.0)
+      google-apis-core (>= 0.4, < 2.a)
+    google-apis-pubsub_v1 (0.12.0)
+      google-apis-core (>= 0.4, < 2.a)
+    google-apis-sqladmin_v1beta4 (0.24.0)
+      google-apis-core (>= 0.4, < 2.a)
+    google-apis-storage_v1 (0.11.0)
+      google-apis-core (>= 0.4, < 2.a)
+    google-cloud-env (1.5.0)
+      faraday (>= 0.17.3, < 2.0)
     google-protobuf (3.19.2)
     google-protobuf (3.19.2-x86_64-darwin)
     google-protobuf (3.19.2-x86_64-linux)
@@ -206,13 +239,13 @@ GEM
       grpc (~> 1.27)
     googleapis-common-protos-types (1.3.0)
       google-protobuf (~> 3.14)
-    googleauth (0.6.7)
-      faraday (~> 0.12)
+    googleauth (1.1.1)
+      faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.4, < 3.0)
       memoist (~> 0.16)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
-      signet (~> 0.7)
+      signet (>= 0.16, < 2.a)
     grpc (1.42.0)
       google-protobuf (~> 3.18)
       googleapis-common-protos-types (~> 1.0)
@@ -249,7 +282,7 @@ GEM
     json_pure (2.6.1)
     jsonpath (1.1.0)
       multi_json
-    jwt (2.2.1)
+    jwt (2.3.0)
     kramdown (2.3.1)
       rexml
     kramdown-parser-gfm (1.1.0)
@@ -273,12 +306,13 @@ GEM
       nokogiri (>= 1.5.9)
     machinist (1.0.6)
     membrane (1.1.0)
-    memoist (0.16.0)
+    memoist (0.16.2)
     method_source (1.0.0)
     middleware (0.1.0)
     mime-types (3.4.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2021.1115)
+    mime-types-data (3.2022.0105)
+    mini_mime (1.1.2)
     mini_portile2 (2.8.0)
     minitest (5.15.0)
     ms_rest (0.6.4)
@@ -310,7 +344,7 @@ GEM
     nokogiri (1.13.3-x86_64-linux)
       racc (~> 1.4)
     oj (3.13.11)
-    os (1.0.1)
+    os (1.1.4)
     palm_civet (1.1.0)
     parallel (1.21.0)
     parallel_tests (3.7.3)
@@ -355,9 +389,9 @@ GEM
       ffi (~> 1.0)
     recursive-open-struct (1.1.3)
     regexp_parser (2.2.0)
-    representable (3.0.4)
+    representable (3.1.1)
       declarative (< 0.1.0)
-      declarative-option (< 0.2.0)
+      trailblazer-option (>= 0.1.1, < 0.2.0)
       uber (< 0.2.0)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
@@ -429,9 +463,9 @@ GEM
       pg (>= 0.18.0, != 1.2.0)
       sequel (>= 4.38.0)
     sexp_processor (4.7.0)
-    signet (0.11.0)
-      addressable (~> 2.3)
-      faraday (~> 0.9)
+    signet (0.16.0)
+      addressable (~> 2.8)
+      faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
     simplecov (0.13.0)
@@ -484,7 +518,8 @@ GEM
     thread_safe (0.3.6)
     tilt (2.0.10)
     timecop (0.9.4)
-    timeliness (0.3.10)
+    timeliness (0.4.4)
+    trailblazer-option (0.1.2)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
     uber (0.1.0)
@@ -500,7 +535,8 @@ GEM
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
     webrick (1.7.0)
-    xml-simple (1.1.5)
+    xml-simple (1.1.9)
+      rexml
     yajl-ruby (1.4.1)
     yard (0.9.27)
       webrick (~> 1.7.0)
@@ -520,7 +556,7 @@ DEPENDENCIES
   addressable
   allowy (>= 2.1.0)
   awesome_print
-  azure-storage (= 0.14.0.preview)
+  azure-storage-blob!
   bits_service_client (~> 3.3, >= 3.3.0)
   byebug
   cf-copilot (= 0.0.14)
@@ -535,7 +571,8 @@ DEPENDENCIES
   fog-aliyun
   fog-aws
   fog-azure-rm!
-  fog-google
+  fog-core (~> 2.1.2)
+  fog-google (~> 1.18.0)
   fog-local
   fog-openstack
   googleapis-common-protos (>= 1.3.12)

lib/cloud_controller/blobstore/fog/fog_client.rb

@@ -114,7 +114,7 @@ def blob(key)
       end
 
       def files_for(prefix, _ignored_directory_prefixes=[])
-        if connection.is_a? Fog::Storage::Local::Real
+        if connection.is_a? Fog::Local::Storage::Real
           directory = connection.directories.get(File.join(dir.key, prefix || ''))
           directory ? directory.files : []
         else
@@ -126,7 +126,7 @@ def ensure_bucket_exists
         return if local?
 
         options = { max_keys: 1 }
-        options['limit'] = 1 if connection.service == Fog::Storage::OpenStack
+        options['limit'] = 1 if connection.service == Fog::OpenStack::Storage
         connection.directories.get(@directory_key, options) || connection.directories.create(key: @directory_key, public: false)
       end
 

lib/cloud_controller/blobstore/fog/providers.rb

@@ -1,6 +1,19 @@
+require 'fog/core'
+
+# This can be removed once fog-google fixes
+# https://github.com/fog/fog-google/issues/421, which will allow us to upgrade
+# fog-core to 2.2.4.
+original = Fog::Logger[:deprecation]
+Fog::Logger[:deprecation] = nil
+
 require 'fog/aliyun'
+# Hack until https://github.com/fog/fog-aliyun/pull/155 is merged and released
+Fog::Aliyun::Compute = Fog::Compute::Aliyun
+
 require 'fog/aws'
 require 'fog/local'
 require 'fog/google'
 require 'fog/azurerm'
 require 'fog/openstack'
+
+Fog::Logger[:deprecation] = original

spec/unit/lib/cloud_controller/blobstore/fog/fog_client_spec.rb

@@ -729,21 +729,21 @@ def upload_tmpfile(client, key='abcdef')
           end
 
           it 'gets the bucket' do
-            expect_any_instance_of(Fog::Storage::AWS::Directories).to receive(:get).with(directory_key, max_keys: 1)
+            expect_any_instance_of(Fog::AWS::Storage::Directories).to receive(:get).with(directory_key, max_keys: 1)
             subject.ensure_bucket_exists
           end
 
           context 'the bucket exists' do
             it 'does not create the bucket' do
               subject.ensure_bucket_exists
-              expect_any_instance_of(Fog::Storage::AWS::Directories).not_to receive(:create).with(key: directory_key, public: false)
+              expect_any_instance_of(Fog::AWS::Storage::Directories).not_to receive(:create).with(key: directory_key, public: false)
               subject.ensure_bucket_exists
             end
           end
 
           context 'the bucket does not exist' do
             it 'creates the bucket' do
-              expect_any_instance_of(Fog::Storage::AWS::Directories).to receive(:create).with(key: directory_key, public: false)
+              expect_any_instance_of(Fog::AWS::Storage::Directories).to receive(:create).with(key: directory_key, public: false)
               subject.ensure_bucket_exists
             end
           end