Basic cloud inventory

With this commit we enhance cloud manager so that it now inventories
a couple of the most basic entities: flavors, availability zones and
vms.

Signed-off-by: Miha Pleško <miha.plesko@xlab.si>

Author: Miha Pleško

app/models/manageiq/providers/azure_stack/cloud_manager/vm.rb

@@ -1,5 +1,15 @@
 class ManageIQ::Providers::AzureStack::CloudManager::Vm < ManageIQ::Providers::CloudManager::Vm
 
+  POWER_STATES = {
+    'PowerState/running'      => 'on',
+    'PowerState/starting'     => 'powering_up',
+    'PowerState/stopped'      => 'suspended',
+    'PowerState/stopping'     => 'suspending',
+    'PowerState/deallocated'  => 'off',
+    'PowerState/deallocating' => 'powering_down',
+    'PowerState/unknown'      => 'unknown'
+  }.freeze
+
   def provider_object(connection = nil)
     connection ||= ext_management_system.connect
     # find vm instance via connection and return it
@@ -32,10 +42,8 @@ def raw_suspend
     update_attributes!(:raw_power_state => "suspended")
   end
 
-  # TODO: this method could be the default in a baseclass
   def self.calculate_power_state(raw_power_state)
-    # do some mapping on powerstates
-    # POWER_STATES[raw_power_state.to_s] || "terminated"
-    raw_power_state
+    # https://docs.microsoft.com/en-us/azure/virtual-machines/windows/states-lifecycle
+    POWER_STATES[raw_power_state.to_s] || 'unknown'
   end
 end

app/models/manageiq/providers/azure_stack/inventory/collector.rb

@@ -1,11 +1,42 @@
 class ManageIQ::Providers::AzureStack::Inventory::Collector < ManageIQ::Providers::Inventory::Collector
   require_nested :CloudManager
 
+  def initialize(manager, refresh_target)
+    super(manager, refresh_target)
+    @token = nil
+  end
+
   def azure_resources
-    @azure_resources ||= manager.connect
+    with_shared_token { |token| @azure_resources ||= manager.connect(:token => token) }
+  end
+
+  def azure_compute
+    with_shared_token { |token| @azure_compute ||= manager.connect(:token => token, :service => :Compute) }
   end
 
   def azure_network
-    @azure_network ||= manager.connect(:service => :Network)
+    with_shared_token { |token| @azure_network ||= manager.connect(:token => token, :service => :Network) }
+  end
+
+  def with_shared_token
+    client = yield @token
+    @token ||= client.credentials
+    client
+  end
+
+  def resource_group_name(ems_ref)
+    if (match = ems_ref.match(%r{/subscriptions/[^/]+/resourceGroups/(?<name>[^/]+)/.+}))
+      match[:name].downcase
+    end
+  end
+
+  def resource_group_id(ems_ref)
+    if (match = ems_ref.match(%r{(?<id>/subscriptions/[^/]+/resourceGroups/[^/]+)/.+}))
+      match[:id].downcase
+    end
+  end
+
+  def raw_power_state(instance_view)
+    instance_view&.statuses&.detect { |s| s.code.start_with?('PowerState/') }&.code
   end
 end

app/models/manageiq/providers/azure_stack/inventory/collector/cloud_manager.rb

@@ -2,4 +2,22 @@ class ManageIQ::Providers::AzureStack::Inventory::Collector::CloudManager < Mana
   def resource_groups
     azure_resources.resource_groups.list
   end
+
+  def flavors
+    azure_compute.virtual_machine_sizes.list(manager.provider_region).value
+  end
+
+  def vms
+    if azure_compute.respond_to?(:instance_view)
+      $azure_stack_log.debug("Fetching VMs, then fetching instance view for each")
+      azure_compute.virtual_machines.list_all.each do |vm|
+        vm.instance_view = azure_compute.virtual_machines.instance_view(resource_group_name(vm.id), vm.name)
+      end
+    else
+      $azure_stack_log.debug("Fetching VM ids, then fetching full data for each")
+      azure_resources.resources.list(:filter => "resourceType eq 'Microsoft.Compute/virtualMachines'").map do |vm|
+        azure_compute.virtual_machines.get(resource_group_name(vm.id), vm.name, :expand => 'instanceView')
+      end
+    end
+  end
 end

app/models/manageiq/providers/azure_stack/inventory/parser/cloud_manager.rb

@@ -4,6 +4,9 @@ def parse
     $azure_stack_log.info("#{log_header}...")
 
     resource_groups
+    availability_zones
+    flavors
+    vms
 
     $azure_stack_log.info("#{log_header}...Complete")
   end
@@ -16,4 +19,86 @@ def resource_groups
       )
     end
   end
+
+  def availability_zones
+    persister.availability_zones.build(
+      :ems_ref => 'default',
+      :name    => persister.manager.name
+    )
+  end
+
+  def flavors
+    collector.flavors.each do |flavor|
+      persister.flavors.build(
+        :ems_ref        => flavor.name.downcase,
+        :name           => flavor.name,
+        :cpus           => flavor.number_of_cores,
+        :cpu_cores      => flavor.number_of_cores / vcpus_per_socket(flavor.name),
+        :memory         => flavor.memory_in_mb.megabytes,
+        :root_disk_size => flavor.os_disk_size_in_mb.megabytes,
+        :swap_disk_size => flavor.resource_disk_size_in_mb.megabytes,
+        :enabled        => true
+      )
+    end
+  end
+
+  def vms
+    collector.vms.each do |vm|
+      uid = vm.id.downcase
+      flavor_ref = vm.hardware_profile.vm_size.downcase
+
+      power_state = 'unknown' unless (power_state = collector.raw_power_state(vm.instance_view))
+
+      vm_obj = persister.vms.build(
+        :name              => vm.name,
+        :ems_ref           => uid,
+        :uid_ems           => uid,
+        :vendor            => 'azure_stack',
+        :connection_state  => 'connected',
+        :raw_power_state   => power_state,
+        :location          => vm.location,
+        :availability_zone => persister.availability_zones.lazy_find('default'),
+        :resource_group    => persister.resource_groups.lazy_find(collector.resource_group_id(vm.id)),
+        :flavor            => persister.flavors.lazy_find(flavor_ref)
+      )
+
+      persister.operating_systems.build(
+        :vm_or_template => vm_obj,
+        :product_name   => guest_os(vm)
+      )
+
+      persister.hardwares.build(
+        :vm_or_template  => vm_obj,
+        :cpu_sockets     => persister.flavors.lazy_find(flavor_ref, :key => :cpu_cores),
+        :cpu_total_cores => persister.flavors.lazy_find(flavor_ref, :key => :cpus),
+        :memory_mb       => persister.flavors.find(flavor_ref).memory,
+        :disk_capacity   => persister.flavors.lazy_find(flavor_ref, :key => :swap_disk_size)
+      )
+    end
+  end
+
+  # Fetch full OS info from image (e.g. 'UbuntuServer 16.04 LTS'), fallback to basic info (e.g. 'Linux').
+  def guest_os(vm)
+    if (image_reference = vm.storage_profile&.image_reference) && image_reference&.offer
+      "#{image_reference.offer} #{image_reference.sku.tr('-', ' ')}"
+    else
+      vm.storage_profile.os_disk.os_type
+    end
+  end
+
+  def vcpus_per_socket(flavor_name)
+    # https://docs.microsoft.com/en-us/azure/virtual-machines/windows/acu
+    # Ratio 1 means each socket/core contains one vCPU
+    #       2 means each socket/core contains two vCPUs
+    case flavor_name.to_s
+    when /_(D_v3|Ds_v3|E_v3|Es_v3|M)$/
+      2
+    when /_F\d+s_v2$/
+      2
+    when /_L\d+s_v2$/
+      2
+    else
+      1
+    end
+  end
 end

app/models/manageiq/providers/azure_stack/inventory/persister/definitions/cloud_collections.rb

@@ -2,6 +2,16 @@ module ManageIQ::Providers::AzureStack::Inventory::Persister::Definitions::Cloud
   extend ActiveSupport::Concern
 
   def initialize_cloud_inventory_collections
+    %i[
+      availability_zones
+      hardwares
+      operating_systems
+      flavors
+      vms
+    ].each do |name|
+      add_collection(cloud, name)
+    end
+
     add_resource_groups
   end
 

app/models/manageiq/providers/azure_stack/network_manager/refresher.rb

@@ -1,4 +1,4 @@
 module ManageIQ::Providers
-  class AzureStack::NetworkManager::Refresher < ManageIQ::Providers::BaseManager::ManagerRefresher
+  class AzureStack::NetworkManager::Refresher < ManageIQ::Providers::BaseManager::Refresher
   end
 end

spec/models/manageiq/providers/azure_stack/cloud_manager/vcr_specs/refresher_spec.rb

@@ -1,11 +1,25 @@
+#
+# This inventory test comes with Azure deployment JSON which makes it reproducible on any Azure Stack.
+# VCR cassettes were recorded against a fresh subscription with nothing but that JSON deployed.
+# Please make sure tests stay in sync with the JSON - don't ever manually update VCR cassettes.
+# Instead, modify the JSON and deploy it and then just re-record the cassettes.
+#
+# The JSON deployment is here:
+# ../vcr_fixtures/full-refresh-deployment.json
+#
+
 describe ManageIQ::Providers::AzureStack::CloudManager::Refresher do
   supported_api_versions do |api_version|
     before do
       stub_settings_merge(:ems_refresh => { :azure_stack => refresh_settings }) if refresh_settings
     end
 
     let(:resource_group)  { ResourceGroup.find_by(:name => 'demo-resource-group') }
+    let(:zone)            { AvailabilityZone.find_by(:ems_ref => 'default') }
+    let(:vm)              { Vm.find_by(:name => 'demoVm') }
     let(:security_group)  { SecurityGroup.find_by(:name => 'demoSecurityGroup') }
+    let(:flavor)          { Flavor.find_by(:ems_ref => 'standard_a1') }
+
     let(:saving_strategy) { :recursive }
     let(:saver_strategy)  { :default }
     let(:use_ar)          { true }
@@ -48,12 +62,18 @@ def full_refresh_twice
   def assert_inventory
     assert_table_counts
     assert_resource_group
+    assert_availability_zone
+    assert_specific_flavor
+    assert_specific_vm
     assert_security_group
   end
 
   def assert_table_counts
     expect(ExtManagementSystem.count).to eq(1 + 1) # cloud + network manager
     expect(ResourceGroup.count).to eq(1)
+    expect(AvailabilityZone.count).to eq(1)
+    expect(Vm.count).to eq(1)
+    expect(Flavor.count).to eq(70)
     expect(SecurityGroup.count).to eq(1)
   end
 
@@ -62,6 +82,53 @@ def assert_resource_group
     expect(ems_ref_suffix(resource_group.ems_ref)).to eq('') # prefix is actually resource group ems_ref
   end
 
+  def assert_availability_zone
+    expect(zone).not_to be_nil
+    expect(zone.name).to eq(ems.name)
+  end
+
+  def assert_specific_flavor
+    expect(flavor).not_to be_nil
+    expect(flavor).to have_attributes(
+      :ems_ref        => 'standard_a1',
+      :name           => 'Standard_A1',
+      :cpus           => 1,
+      :cpu_cores      => 1,
+      :memory         => 1.75.gigabytes.round,
+      :root_disk_size => 1023.gigabytes.round,
+      :swap_disk_size => 70.gigabytes.round,
+      :enabled        => true
+    )
+  end
+
+  def assert_specific_vm
+    expect(vm).not_to be_nil
+    expect(ems_ref_suffix(vm.ems_ref)).to match(%r{^/providers/microsoft.compute/virtualmachines/[^/]+$})
+    expect(ems_ref_suffix(vm.uid_ems)).to match(%r{^/providers/microsoft.compute/virtualmachines/[^/]+$})
+
+    expect(vm).to have_attributes(
+      :vendor            => 'azure_stack',
+      :connection_state  => 'connected',
+      :raw_power_state   => 'PowerState/running',
+      :power_state       => 'on',
+      :location          => 'westus',
+      :availability_zone => zone,
+      :resource_group    => resource_group,
+      :flavor            => flavor
+    )
+
+    expect(vm.operating_system).not_to be_nil
+    expect(vm.operating_system.product_name).to eq('UbuntuServer 16.04 LTS')
+
+    expect(vm.hardware).not_to be_nil
+    expect(vm.hardware).to have_attributes(
+      :cpu_sockets     => 1,
+      :cpu_total_cores => 1,
+      :memory_mb       => 1.75.gigabytes.round,
+      :disk_capacity   => 70.gigabytes.round
+    )
+  end
+
   def assert_security_group
     expect(security_group).not_to be_nil
     expect(ems_ref_suffix(security_group.ems_ref)).to match(%r{^/providers/microsoft.network/networksecuritygroups/[^/]+$})

spec/spec_helper.rb

@@ -25,7 +25,7 @@
 def vcr_with_auth(casette)
   VCR.use_cassette('obtain_endpoints', :allow_unused_http_interactions => true) do
     VCR.use_cassette('obtain_token', :allow_playback_repeats => false, :allow_unused_http_interactions => true) do
-      VCR.use_cassette(casette, :allow_unused_http_interactions => false) do
+      VCR.use_cassette(casette, :allow_unused_http_interactions => true) do
         yield
       end
     end