Merge "Provider Config File: Enable loading and merging of provider configs"

Author: Zuul

doc/source/admin/index.rst

@@ -113,6 +113,7 @@ instance for these kind of workloads.
    ports-with-resource-requests
    virtual-persistent-memory
    emulated-tpm
+   managing-resource-providers
 
 
 Additional guides

doc/source/admin/managing-resource-providers.rst

@@ -0,0 +1,216 @@
+==============================================
+Managing Resource Providers Using Config Files
+==============================================
+
+In order to facilitate management of resource provider information in the
+Placement API, Nova provides `a method`__ for admins to add custom inventory
+and traits to resource providers using YAML files.
+
+__ https://specs.openstack.org/openstack/nova-specs/specs/ussuri/approved/provider-config-file.html
+
+.. note::
+
+    Only ``CUSTOM_*`` resource classes and traits may be managed this way.
+
+Placing Files
+-------------
+
+Nova-compute will search for ``*.yaml`` files in the path specified in
+:oslo.config:option:`compute.provider_config_location`. These files will be
+loaded and validated for errors on nova-compute startup. If there are any
+errors in the files, nova-compute will fail to start up.
+
+Administrators should ensure that provider config files have appropriate
+permissions and ownership. See the `specification`__ and `admin guide`__
+for more details.
+
+__ https://specs.openstack.org/openstack/nova-specs/specs/ussuri/approved/provider-config-file.html
+__ https://docs.openstack.org/nova/latest/admin/managing-resource-providers.html
+
+.. note::
+
+    The files are loaded once at nova-compute startup and any changes or new
+    files will not be recognized until the next nova-compute startup.
+
+Examples
+--------
+
+Resource providers to target can be identified by either UUID or name. In
+addition, the value ``$COMPUTE_NODE`` can be used in the UUID field to
+identify all nodes managed by the service.
+
+If an entry does not include any additional inventory or traits, it will be
+logged at load time but otherwise ignored. In the case of a resource provider
+being identified by both ``$COMPUTE_NODE`` and individual UUID/name, the
+values in the ``$COMPUTE_NODE`` entry will be ignored for *that provider* only
+if the explicit entry includes inventory or traits.
+
+.. note::
+
+    In the case that a resource provider is identified more than once by
+    explicit UUID/name, the nova-compute service will fail to start. This
+    is a global requirement across all supplied ``provider.yaml`` files.
+
+.. code-block:: yaml
+
+    meta:
+      schema_version: '1.0'
+    providers:
+      - identification:
+          name: 'EXAMPLE_RESOURCE_PROVIDER'
+          # Additional valid identification examples:
+          # uuid: '$COMPUTE_NODE'
+          # uuid: '5213b75d-9260-42a6-b236-f39b0fd10561'
+        inventories:
+          additional:
+            - CUSTOM_EXAMPLE_RESOURCE_CLASS:
+                total: 100
+                reserved: 0
+                min_unit: 1
+                max_unit: 10
+                step_size: 1
+                allocation_ratio: 1.0
+        traits:
+          additional:
+            - 'CUSTOM_EXAMPLE_TRAIT'
+
+Schema Example
+--------------
+.. code-block:: yaml
+
+  type: object
+  properties:
+    # This property is used to track where the provider.yaml file originated.
+    # It is reserved for internal use and should never be set in a provider.yaml
+    # file supplied by an end user.
+    __source_file:
+      not: {}
+    meta:
+      type: object
+      properties:
+        # Version ($Major, $minor) of the schema must successfully parse
+        # documents conforming to ($Major, 0..N). Any breaking schema change
+        # (e.g. removing fields, adding new required fields, imposing a stricter
+        # pattern on a value, etc.) must bump $Major.
+        schema_version:
+          type: string
+          pattern: '^1\.([0-9]|[1-9][0-9]+)$'
+      required:
+        - schema_version
+      additionalProperties: true
+    providers:
+      type: array
+      items:
+        type: object
+        properties:
+          identification:
+            $ref: '#/provider_definitions/provider_identification'
+          inventories:
+            $ref: '#/provider_definitions/provider_inventories'
+          traits:
+            $ref: '#/provider_definitions/provider_traits'
+        required:
+          - identification
+        additionalProperties: true
+  required:
+    - meta
+  additionalProperties: true
+
+  provider_definitions:
+    provider_identification:
+      # Identify a single provider to configure. Exactly one identification
+      # method should be used. Currently `uuid` or `name` are supported, but
+      # future versions may support others.
+      # The uuid can be set to the sentinel value `$COMPUTE_NODE` which will
+      # cause the consuming compute service to apply the configuration to
+      # to all compute node root providers it manages that are not otherwise
+      # specified using a uuid or name.
+      type: object
+      properties:
+        uuid:
+          oneOf:
+              # TODO(sean-k-mooney): replace this with type uuid when we can depend
+              # on a version of the jsonschema lib that implements draft 8 or later
+              # of the jsonschema spec.
+            - type: string
+              pattern: '^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$'
+            - type: string
+              const: '$COMPUTE_NODE'
+        name:
+          type: string
+          minLength: 1
+      # This introduces the possibility of an unsupported key name being used to
+      # get by schema validation, but is necessary to support forward
+      # compatibility with new identification methods. This should be checked
+      # after schema validation.
+      minProperties: 1
+      maxProperties: 1
+      additionalProperties: false
+    provider_inventories:
+      # Allows the admin to specify various adjectives to create and manage
+      # providers' inventories. This list of adjectives can be extended in the
+      # future as the schema evolves to meet new use cases. As of v1.0, only one
+      # adjective, `additional`, is supported.
+      type: object
+      properties:
+        additional:
+          type: array
+          items:
+            patternProperties:
+              # Allows any key name matching the resource class pattern,
+              # check to prevent conflicts with virt driver owned resouces classes
+              # will be done after schema validation.
+              ^[A-Z0-9_]{1,255}$:
+                type: object
+                properties:
+                  # Any optional properties not populated will be given a default value by
+                  # placement. If overriding a pre-existing provider values will not be
+                  # preserved from the existing inventory.
+                  total:
+                    type: integer
+                  reserved:
+                    type: integer
+                  min_unit:
+                    type: integer
+                  max_unit:
+                    type: integer
+                  step_size:
+                    type: integer
+                  allocation_ratio:
+                    type: number
+                required:
+                  - total
+                # The defined properties reflect the current placement data
+                # model. While defining those in the schema and not allowing
+                # additional properties means we will need to bump the schema
+                # version if they change, that is likely to be part of a large
+                # change that may have other impacts anyway. The benefit of
+                # stricter validation of property names outweighs the (small)
+                # chance of having to bump the schema version as described above.
+                additionalProperties: false
+            # This ensures only keys matching the pattern above are allowed
+            additionalProperties: false
+      additionalProperties: true
+    provider_traits:
+      # Allows the admin to specify various adjectives to create and manage
+      # providers' traits. This list of adjectives can be extended in the
+      # future as the schema evolves to meet new use cases. As of v1.0, only one
+      # adjective, `additional`, is supported.
+      type: object
+      properties:
+        additional:
+          type: array
+          items:
+            # Allows any value matching the trait pattern here, additional
+            # validation will be done after schema validation.
+            type: string
+            pattern: '^[A-Z0-9_]{1,255}$'
+      additionalProperties: true
+
+.. note::
+
+    When creating a ``provider.yaml`` config file it is recommended to use the
+    schema provided by nova to validate the config using a simple jsonschema
+    validator rather than starting the nova compute agent to enable faster
+    iteration.
+

nova/compute/resource_tracker.py

@@ -30,6 +30,7 @@
 
 from nova.compute import claims
 from nova.compute import monitors
+from nova.compute import provider_config
 from nova.compute import stats as compute_stats
 from nova.compute import task_states
 from nova.compute import utils as compute_utils
@@ -112,6 +113,11 @@ def __init__(self, host, driver, reportclient=None):
         # and value of this sub-dict is a set of Resource obj
         self.assigned_resources = collections.defaultdict(
             lambda: collections.defaultdict(set))
+        # Retrieves dict of provider config data. This can fail with
+        # nova.exception.ProviderConfigException if invalid or conflicting
+        # data exists in the provider config files.
+        self.provider_configs = provider_config.get_provider_configs(
+            CONF.compute.provider_config_location)
         # Set of ids for providers identified in provider config files that
         # are not found on the provider tree. These are tracked to facilitate
         # smarter logging.
@@ -1155,6 +1161,9 @@ def _update_to_placement(self, context, compute_node, startup):
 
         self.provider_tree = prov_tree
 
+        # This merges in changes from the provider config files loaded in init
+        self._merge_provider_configs(self.provider_configs, prov_tree)
+
         # Flush any changes. If we processed ReshapeNeeded above, allocs is not
         # None, and this will hit placement's POST /reshaper route.
         self.reportclient.update_from_provider_tree(context, prov_tree,
@@ -1714,6 +1723,7 @@ def _merge_provider_configs(self, provider_configs, provider_tree):
         :param provider_tree: The provider tree to be updated in place
         """
         processed_providers = {}
+        provider_custom_traits = {}
         for uuid_or_name, provider_data in provider_configs.items():
             additional_traits = provider_data.get(
                 "traits", {}).get("additional", [])
@@ -1758,10 +1768,23 @@ def _merge_provider_configs(self, provider_configs, provider_tree):
                             'current_uuid': current_uuid
                         }
                     )
-                processed_providers[current_uuid] = source_file_name
+
+                # NOTE(sean-k-mooney): since each provider should be processed
+                # at most once if a provider has custom traits they were
+                # set either in previous iteration, the virt driver or via the
+                # the placement api. As a result we must ignore them when
+                # checking for duplicate traits so we construct a set of the
+                # existing custom traits.
+                if current_uuid not in provider_custom_traits:
+                    provider_custom_traits[current_uuid] = {
+                        trait for trait in provider.traits
+                        if trait.startswith('CUSTOM')
+                    }
+                existing_custom_traits = provider_custom_traits[current_uuid]
 
                 if additional_traits:
                     intersect = set(provider.traits) & set(additional_traits)
+                    intersect -= existing_custom_traits
                     if intersect:
                         invalid = ','.join(intersect)
                         raise ValueError(_(
@@ -1797,6 +1820,8 @@ def _merge_provider_configs(self, provider_configs, provider_tree):
                     provider_tree.update_inventory(
                         provider.uuid, merged_inventory)
 
+                processed_providers[current_uuid] = source_file_name
+
     def _get_providers_to_update(self, uuid_or_name, provider_tree,
                                  source_file):
         """Identifies the providers to be updated.

nova/conf/compute.py

@@ -967,6 +967,19 @@
 
 * -1 means unlimited
 * Any integer >= 0 represents the maximum allowed
+"""),
+    cfg.StrOpt('provider_config_location',
+        default='/etc/nova/provider_config/',
+        help="""
+Location of YAML files containing resource provider configuration data.
+
+These files allow the operator to specify additional custom inventory and
+traits to assign to one or more resource providers.
+
+Additional documentation is available here:
+
+  https://docs.openstack.org/nova/latest/admin/managing-resource-providers.html
+
 """),
 ]