Add storage cli param to cc deployment updater (#583)

Author: kathap

jobs/cc_deployment_updater/spec

@@ -27,6 +27,10 @@ templates:
   resource_pool_ca_cert.pem.erb: config/certs/resource_pool_ca_cert.pem
   droplets_ca_cert.pem.erb: config/certs/droplets_ca_cert.pem
   buildpacks_ca_cert.pem.erb: config/certs/buildpacks_ca_cert.pem
+  storage_cli_config_droplets.json.erb: config/storage_cli_config_droplets.json
+  storage_cli_config_packages.json.erb: config/storage_cli_config_packages.json
+  storage_cli_config_buildpacks.json.erb: config/storage_cli_config_buildpacks.json
+  storage_cli_config_resource_pool.json.erb: config/storage_cli_config_resource_pool.json
 
 packages:
   - capi_utils

jobs/cc_deployment_updater/templates/cloud_controller_ng.yml.erb

@@ -152,6 +152,11 @@ maximum_health_check_timeout: <%= link("cloud_controller_internal").p("cc.maximu
 
 stacks_file: /var/vcap/jobs/cloud_controller_ng/config/stacks.yml
 
+storage_cli_config_file_droplets: /var/vcap/jobs/cc_deployment_updater/config/storage_cli_config_droplets.json
+storage_cli_config_file_buildpacks: /var/vcap/jobs/cc_deployment_updater/config/storage_cli_config_buildpacks.json
+storage_cli_config_file_packages: /var/vcap/jobs/cc_deployment_updater/config/storage_cli_config_packages.json
+storage_cli_config_file_resource_pool: /var/vcap/jobs/cc_deployment_updater/config/storage_cli_config_resource_pool.json
+
 resource_pool:
   blobstore_type: <%= link("cloud_controller_internal").p("cc.resource_pool.blobstore_type") %>
   webdav_config:
@@ -166,6 +171,8 @@ resource_pool:
   minimum_size: <%= link("cloud_controller_internal").p("cc.resource_pool.minimum_size") %>
   maximum_size: <%= link("cloud_controller_internal").p("cc.resource_pool.maximum_size") %>
   resource_directory_key: <%= link("cloud_controller_internal").p("cc.resource_pool.resource_directory_key") %>
+  blobstore_provider: <%= link("cloud_controller_internal").p("cc.resource_pool.blobstore_provider") %>
+  connection_config: <%= link("cloud_controller_internal").p("cc.resource_pool.connection_config", {}).to_json %>
   <% link("cloud_controller_internal").if_p("cc.resource_pool.cdn") do %>
   cdn:
     uri: <%= link("cloud_controller_internal").p("cc.resource_pool.cdn.uri") %>
@@ -188,6 +195,8 @@ packages:
     ca_cert_path: "/var/vcap/jobs/cc_deployment_updater/config/certs/packages_ca_cert.pem"
     <% end %>
   app_package_directory_key: <%= link("cloud_controller_internal").p("cc.packages.app_package_directory_key") %>
+  blobstore_provider: <%= link("cloud_controller_internal").p("cc.packages.blobstore_provider") %>
+  connection_config: <%= link("cloud_controller_internal").p("cc.packages.connection_config", {}).to_json %>
   max_package_size: <%= link("cloud_controller_internal").p("cc.packages.max_package_size") %>
   <% link("cloud_controller_internal").if_p("cc.packages.cdn") do %>
   cdn:
@@ -212,6 +221,8 @@ droplets:
     ca_cert_path: "/var/vcap/jobs/cc_deployment_updater/config/certs/droplets_ca_cert.pem"
     <% end %>
   droplet_directory_key: <%= link("cloud_controller_internal").p("cc.droplets.droplet_directory_key") %>
+  blobstore_provider: <%= link("cloud_controller_internal").p("cc.droplets.blobstore_provider") %>
+  connection_config: <%= link("cloud_controller_internal").p("cc.droplets.connection_config", {}).to_json %>
   <% link("cloud_controller_internal").if_p("cc.droplets.cdn") do %>
   cdn:
     uri: <%= link("cloud_controller_internal").p("cc.droplets.cdn.uri") %>
@@ -234,6 +245,8 @@ buildpacks:
     ca_cert_path: "/var/vcap/jobs/cc_deployment_updater/config/certs/buildpacks_ca_cert.pem"
     <% end %>
   buildpack_directory_key: <%= link("cloud_controller_internal").p("cc.buildpacks.buildpack_directory_key") %>
+  blobstore_provider: <%= link("cloud_controller_internal").p("cc.buildpacks.blobstore_provider") %>
+  connection_config: <%= link("cloud_controller_internal").p("cc.buildpacks.connection_config", {}).to_json %>
   <% link("cloud_controller_internal").if_p("cc.buildpacks.cdn") do %>
   cdn:
     uri: <%= link("cloud_controller_internal").p("cc.buildpacks.cdn.uri") %>

jobs/cc_deployment_updater/templates/storage_cli_config_buildpacks.json.erb

@@ -0,0 +1,50 @@
+<%
+require "json"
+
+ # Ensure Azure CLI connection_config has a default timeout if none is set
+def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41)
+  cfg = (connection_cfg || {}).dup
+  if blobstore_type == 'storage_cli'
+    if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty?
+      cfg['put_timeout_in_seconds'] = default_seconds.to_s
+    end
+  end
+  cfg
+end
+
+# helper: add key only when value is present
+def add(h, key, val)
+  return if val.nil?
+  return if val.respond_to?(:empty?) && val.empty?
+  h[key] = val
+end
+
+l = link("cloud_controller_internal")
+
+scope = "cc.buildpacks.connection_config"
+provider = l.p("cc.buildpacks.blobstore_provider", nil)
+
+if provider != "AzureRM"
+  options = {} # for now: all non-azure providers output an empty JSON object
+else
+  options = {}
+  options["provider"] = provider
+  options["account_name"]   = l.p("#{scope}.azure_storage_account_name")
+  options["container_name"] = l.p("#{scope}.container_name")
+  add(options, "account_key",        l.p("#{scope}.azure_storage_access_key"))
+  add(options, "environment",        l.p("#{scope}.environment", "AzureCloud"))
+  add(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil))
+
+  # optional passthrough for extra storage-cli flags
+  begin
+    custom = l.p("#{scope}.custom", {})
+    if custom.respond_to?(:each)
+      custom.each { |k, v| add(options, k.to_s, v) }
+    end
+  rescue
+    # ignore if property not defined
+  end
+  options = cli_cfg_with_default_timeout(options, 'storage_cli')
+end
+-%>
+<%= JSON.pretty_generate(options) %>

jobs/cc_deployment_updater/templates/storage_cli_config_droplets.json.erb

@@ -0,0 +1,50 @@
+<%
+require "json"
+
+ # Ensure Azure CLI connection_config has a default timeout if none is set
+def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41)
+  cfg = (connection_cfg || {}).dup
+  if blobstore_type == 'storage_cli'
+    if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty?
+      cfg['put_timeout_in_seconds'] = default_seconds.to_s
+    end
+  end
+  cfg
+end
+
+# helper: add key only when value is present
+def add(h, key, val)
+  return if val.nil?
+  return if val.respond_to?(:empty?) && val.empty?
+  h[key] = val
+end
+
+l = link("cloud_controller_internal")
+
+scope = "cc.droplets.connection_config"
+provider = l.p("cc.droplets.blobstore_provider", nil)
+
+if provider != "AzureRM"
+  options = {} # for now: all non-azure providers output an empty JSON object
+else
+  options = {}
+  options["provider"] = provider
+  options["account_name"]   = l.p("#{scope}.azure_storage_account_name")
+  options["container_name"] = l.p("#{scope}.container_name")
+  add(options, "account_key",        l.p("#{scope}.azure_storage_access_key"))
+  add(options, "environment",        l.p("#{scope}.environment", "AzureCloud"))
+  add(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil))
+
+  # optional passthrough for extra storage-cli flags
+  begin
+    custom = l.p("cc.droplets.connection_config.custom", {})
+    if custom.respond_to?(:each)
+      custom.each { |k, v| add(options, k.to_s, v) }
+    end
+  rescue
+    # ignore if property not defined
+  end
+  options = cli_cfg_with_default_timeout(options, 'storage_cli')
+end
+-%>
+<%= JSON.pretty_generate(options) %>

jobs/cc_deployment_updater/templates/storage_cli_config_packages.json.erb

@@ -0,0 +1,50 @@
+<%
+require "json"
+
+ # Ensure Azure CLI connection_config has a default timeout if none is set
+def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41)
+  cfg = (connection_cfg || {}).dup
+  if blobstore_type == 'storage_cli'
+    if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty?
+      cfg['put_timeout_in_seconds'] = default_seconds.to_s
+    end
+  end
+  cfg
+end
+
+# helper: add key only when value is present
+def add(h, key, val)
+  return if val.nil?
+  return if val.respond_to?(:empty?) && val.empty?
+  h[key] = val
+end
+
+l = link("cloud_controller_internal")
+
+scope = "cc.packages.connection_config"
+provider = l.p("cc.packages.blobstore_provider", nil)
+
+if provider != "AzureRM"
+  options = {} # for now: all non-azure providers output an empty JSON object
+else
+  options = {}
+  options["provider"] = provider
+  options["account_name"]   = l.p("#{scope}.azure_storage_account_name")
+  options["container_name"] = l.p("#{scope}.container_name")
+  add(options, "account_key",        l.p("#{scope}.azure_storage_access_key"))
+  add(options, "environment",        l.p("#{scope}.environment", "AzureCloud"))
+  add(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil))
+
+  # optional passthrough for extra storage-cli flags
+  begin
+    custom = l.p("#{scope}.custom", {})
+    if custom.respond_to?(:each)
+      custom.each { |k, v| add(options, k.to_s, v) }
+    end
+  rescue
+    # ignore if property not defined
+  end
+  options = cli_cfg_with_default_timeout(options, 'storage_cli')
+end
+-%>
+<%= JSON.pretty_generate(options) %>

jobs/cc_deployment_updater/templates/storage_cli_config_resource_pool.json.erb

@@ -0,0 +1,50 @@
+<%
+require "json"
+
+ # Ensure Azure CLI connection_config has a default timeout if none is set
+def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41)
+  cfg = (connection_cfg || {}).dup
+  if blobstore_type == 'storage_cli'
+    if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty?
+      cfg['put_timeout_in_seconds'] = default_seconds.to_s
+    end
+  end
+  cfg
+end
+
+# helper: add key only when value is present
+def add(h, key, val)
+  return if val.nil?
+  return if val.respond_to?(:empty?) && val.empty?
+  h[key] = val
+end
+
+l = link("cloud_controller_internal")
+
+scope = "cc.resource_pool.connection_config"
+provider = l.p("cc.resource_pool.blobstore_provider", nil)
+
+if provider != "AzureRM"
+  options = {} # for now: all non-azure providers output an empty JSON object
+else
+  options = {}
+  options["provider"] = provider
+  options["account_name"]   = l.p("#{scope}.azure_storage_account_name")
+  options["container_name"] = l.p("#{scope}.container_name")
+  add(options, "account_key",        l.p("#{scope}.azure_storage_access_key"))
+  add(options, "environment",        l.p("#{scope}.environment", "AzureCloud"))
+  add(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil))
+
+  # optional passthrough for extra storage-cli flags
+  begin
+    custom = l.p("#{scope}.custom", {})
+    if custom.respond_to?(:each)
+      custom.each { |k, v| add(options, k.to_s, v) }
+    end
+  rescue
+    # ignore if property not defined
+  end
+  options = cli_cfg_with_default_timeout(options, 'storage_cli')
+end
+-%>
+<%= JSON.pretty_generate(options) %>

spec/cc_deployment_updater/storage_cli_config_jsons_spec.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require 'rspec'
+require 'yaml'
+require 'bosh/template/test'
+
+TEMPLATES = {
+  droplets: ['config/storage_cli_config_droplets.json', %w[cc droplets connection_config]],
+  buildpacks: ['config/storage_cli_config_buildpacks.json', %w[cc buildpacks connection_config]],
+  packages: ['config/storage_cli_config_packages.json', %w[cc packages connection_config]],
+  resource_pool: ['config/storage_cli_config_resource_pool.json', %w[cc resource_pool connection_config]]
+}.freeze
+
+module Bosh
+  module Template
+    module Test
+      RSpec.describe 'storage-cli JSON templates (cc_deployment_updater)' do
+        let(:release_path) { File.join(File.dirname(__FILE__), '../..') }
+        let(:release)      { ReleaseDir.new(release_path) }
+        let(:job)          { release.job('cc_deployment_updater') }
+
+        let(:link_props) do
+          {
+            'cc' => {
+              'droplets' => { 'connection_config' => {}, 'blobstore_provider' => 'S3' },
+              'buildpacks' => { 'connection_config' => {}, 'blobstore_provider' => 'S3' },
+              'packages' => { 'connection_config' => {}, 'blobstore_provider' => 'S3' },
+              'resource_pool' => { 'connection_config' => {}, 'blobstore_provider' => 'S3' }
+            }
+          }
+        end
+
+        let(:cc_link) do
+          Bosh::Template::Test::Link.new(
+            name: 'cloud_controller_internal',
+            properties: link_props
+          )
+        end
+
+        let(:links) { [cc_link] }
+        let(:props) { {} }
+
+        def set(hash, path, value)
+          cursor = hash
+          path[0..-2].each { |key| cursor = (cursor[key] ||= {}) }
+          cursor[path.last] = value
+        end
+
+        TEMPLATES.each do |scope, (template_path, keypath)|
+          describe template_path do
+            let(:template) { job.template(template_path) }
+
+            context "when provider is AzureRM for #{scope}" do
+              before do
+                link_props['cc'][scope.to_s]['blobstore_provider'] = 'AzureRM'
+              end
+
+              it 'renders and normalizes put_timeout_in_seconds to "41" when blank' do
+                set(link_props, keypath, {
+                      'provider' => 'AzureRM',
+                      'azure_storage_account_name' => 'acc',
+                      'azure_storage_access_key' => 'key',
+                      'container_name' => 'cont',
+                      'put_timeout_in_seconds' => ''
+                    })
+
+                json = YAML.safe_load(template.render(props, consumes: links))
+                expect(json).to include(
+                  'provider' => 'AzureRM',
+                  'account_name' => 'acc',
+                  'account_key' => 'key',
+                  'container_name' => 'cont',
+                  'put_timeout_in_seconds' => '41'
+                )
+              end
+
+              it 'keeps existing put_timeout_in_seconds when provided' do
+                set(link_props, keypath, {
+                      'provider' => 'AzureRM',
+                      'azure_storage_account_name' => 'acc',
+                      'azure_storage_access_key' => 'key',
+                      'container_name' => 'cont',
+                      'put_timeout_in_seconds' => '7'
+                    })
+
+                json = YAML.safe_load(template.render(props, consumes: links))
+                expect(json['put_timeout_in_seconds']).to eq('7')
+              end
+            end
+
+            context "when provider is non-Azure for #{scope}" do
+              it 'renders {}' do
+                json = YAML.safe_load(template.render(props, consumes: links))
+                expect(json).to eq({})
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end