Implement online attribute filter; pass model context for use by internal filters

Author: rakillen

core/src/main/python/validate.py

@@ -122,7 +122,7 @@ def __perform_model_file_validation(model_file_name, model_context):
         # substitute variables before filtering
         variables.substitute(model_dictionary, variable_map, model_context)
         # apply filters to merged model
-        if filter_helper.apply_filters(model_dictionary, "validate"):
+        if filter_helper.apply_filters(model_dictionary, "validate", model_context):
             # persist model after filtering
             cla_helper.persist_model(model_context, model_dictionary)
 

core/src/main/python/wlsdeploy/tool/util/filter_helper.py

@@ -22,26 +22,26 @@
 }
 
 
-def apply_filters(model, tool_type, model_context=None):
+def apply_filters(model, tool_type, model_context):
     """
     Apply any filters configured for the specified tool type to the specified model.
     :param model: the model to be filtered
     :param tool_type: the name of the filter tool type
-    :param model_context: optional, used to find target filters
+    :param model_context: used to find target filters
     :return: True if any filter was applied, False otherwise
     :raises: BundleAwareException of the specified type: if an error occurs
     """
     _method_name = 'apply_filters'
-    global __filter_file_location
 
-    __filter_file_location = path_utils.find_config_path('model_filters.json')
+    __filter_file_location = None
     filter_applied = False
 
     try:
         filters_dictionary = {}
 
         # if target specified in model context, use the filters from target config
         if model_context and model_context.get_target():
+            __filter_file_location = model_context.get_target_configuration_file()
             filters_dictionary = model_context.get_target_configuration().get_model_filters()
             target_path = os.path.join('targets', model_context.get_target())
 
@@ -53,15 +53,18 @@ def apply_filters(model, tool_type, model_context=None):
                         filter_path = target_path + filter_path.replace(TARGET_CONFIG_TOKEN, '')
                         current_filter['path'] = path_utils.find_config_path(filter_path)
 
-        elif os.path.isfile(__filter_file_location):
-            filters_dictionary = FileToPython(__filter_file_location).parse()
         else:
-            __logger.info('WLSDPLY-20017', __filter_file_location, class_name=__class_name, method_name=_method_name)
+            __filter_file_location = path_utils.find_config_path('model_filters.json')
+            if os.path.isfile(__filter_file_location):
+                filters_dictionary = FileToPython(__filter_file_location).parse()
+            else:
+                __logger.info('WLSDPLY-20017', __filter_file_location, class_name=__class_name,
+                              method_name=_method_name)
 
         if tool_type in filters_dictionary:
             filter_list = filters_dictionary[tool_type]
             for filter in filter_list:
-                filter_applied = _apply_filter(model, filter) or filter_applied
+                filter_applied = _apply_filter(model, filter, model_context, __filter_file_location) or filter_applied
         else:
             __logger.info('WLSDPLY-20016', tool_type, __filter_file_location, class_name=__class_name,
                           method_name=_method_name)
@@ -72,34 +75,39 @@ def apply_filters(model, tool_type, model_context=None):
     return filter_applied
 
 
-def _apply_filter(model, the_filter):
+def _apply_filter(model, the_filter, model_context, filter_file_location):
     """
     Apply the specified filter to the specified model.
     :param model: the model to be filtered
     :param the_filter: a dictionary containing the filter parameters
+    :param model_context: may be used by internal (ID) filters
+    :param filter_file_location: used for logging
     :return: True if the specified filter was applied, False otherwise
     :raises: BundleAwareException of the specified type: if an error occurs
     """
     _method_name = '_apply_filter'
-    global __filter_file_location
 
     id = dictionary_utils.get_element(the_filter, 'id')
     if id is not None:
-        return _apply_id_filter(model, id)
+        __logger.info('WLSDPLY-20034', id, class_name=__class_name, method_name=_method_name)
+        return _apply_id_filter(model, id, model_context)
 
     path = dictionary_utils.get_element(the_filter, 'path')
     if path is not None:
+        name = dictionary_utils.get_element(the_filter, 'name')
+        __logger.info('WLSDPLY-20035', name, class_name=__class_name, method_name=_method_name)
         return _apply_path_filter(model, path)
 
-    __logger.severe('WLSDPLY-20019', str(__filter_file_location), class_name=__class_name, method_name=_method_name)
+    __logger.severe('WLSDPLY-20019', str(filter_file_location), class_name=__class_name, method_name=_method_name)
     return False
 
 
-def _apply_id_filter(model, id):
+def _apply_id_filter(model, id, model_context):
     """
     Apply the specified ID filter to the specified model.
     :param model: the model to be filtered
     :param id: the ID of the filter to be applied
+    :param model_context: may be used by filters
     :return: True if the specified filter was applied, False otherwise
     :raises: BundleAwareException of the specified type: if an error occurs
     """
@@ -110,7 +118,7 @@ def _apply_id_filter(model, id):
         __logger.severe('WLSDPLY-20020', str(id), class_name=__class_name, method_name=_method_name)
         return False
     else:
-        filter_method(model)
+        filter_method(model, model_context)
         return True
 
 

core/src/main/python/wlsdeploy/tool/util/filters/model_traverse.py

@@ -0,0 +1,119 @@
+# Copyright (c) 2021, Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
+
+from wlsdeploy.aliases.aliases import Aliases
+from wlsdeploy.aliases.location_context import LocationContext
+from wlsdeploy.aliases.model_constants import DOMAIN_INFO
+from wlsdeploy.aliases.model_constants import RESOURCES
+from wlsdeploy.aliases.model_constants import TOPOLOGY
+from wlsdeploy.logging.platform_logger import PlatformLogger
+
+_class_name = 'ModelTraverse'
+
+
+class ModelTraverse:
+    """
+    This class traverses all model folders and attributes.
+    Sub-classes can override specific methods to avoid re-implementing the recursive traversal.
+    """
+    def __init__(self, model_context, wlst_mode, exception_type):
+        self._aliases = Aliases(model_context=model_context, wlst_mode=wlst_mode, exception_type=exception_type)
+        self._logger = PlatformLogger('wlsdeploy.model_traverse')
+
+    def traverse_model(self, root_dict):
+        section_names = [DOMAIN_INFO, TOPOLOGY, RESOURCES]
+        for section_name in section_names:
+            self.traverse_section(section_name, root_dict,
+                                  self._aliases.get_model_section_top_level_folder_names(section_name))
+
+    def traverse_section(self, model_section_key, model_dict, valid_section_folders):
+        if model_section_key not in model_dict.keys():
+            return
+
+        # only specific top-level sections have attributes
+        attribute_location = self._aliases.get_model_section_attribute_location(model_section_key)
+        valid_attr_infos = []
+
+        if attribute_location is not None:
+            valid_attr_infos = self._aliases.get_model_attribute_names_and_types(attribute_location)
+
+        model_section_dict = model_dict[model_section_key]
+        for section_dict_key, section_dict_value in model_section_dict.iteritems():
+            # section_dict_key is the name of a folder or attribute in the section.
+
+            if section_dict_key in valid_attr_infos:
+                # section_dict_key is the name of an attribute in the section
+                self.traverse_attribute(model_section_dict, section_dict_key, attribute_location)
+
+            elif section_dict_key in valid_section_folders:
+                # section_dict_key is a folder under the model section
+
+                # Append section_dict_key to location context
+                model_location = LocationContext()
+                model_location.append_location(section_dict_key)
+
+                # Call self.traverse_folder() passing in section_dict_value as the model_node to process
+                self.traverse_folder(section_dict_value, model_location)
+
+    def traverse_folder(self, model_node, model_location):
+        """
+        Traverse a folder that may have named sub-folders (such as Server), artificial named sub-folders (such
+        as a security provider type), or its own sub-folders and attributes (such as .
+        """
+
+        # avoid checking folder type if is_artificial_type_folder
+        is_multiple = not self._aliases.is_artificial_type_folder(model_location) and \
+            (self._aliases.supports_multiple_mbean_instances(model_location) or
+                self._aliases.requires_artificial_type_subfolder_handling(model_location))
+
+        if is_multiple:
+            for name in model_node:
+                expanded_name = name
+                new_location = LocationContext(model_location)
+                name_token = self._aliases.get_name_token(new_location)
+                if name_token is not None:
+                    new_location.add_name_token(name_token, expanded_name)
+                value_dict = model_node[name]
+                self.traverse_node(value_dict, new_location)
+        else:
+            name_token = self._aliases.get_name_token(model_location)
+            if name_token is not None:
+                name = model_location.get_name_for_token(name_token)
+                if name is None:
+                    name = '%s-0' % name_token
+                model_location.add_name_token(name_token, name)
+            self.traverse_node(model_node, model_location)
+
+    def traverse_node(self, model_node, model_location):
+        """
+        Traverse a node that contains only attributes, non-named sub-folders, and artificial type folders.
+        """
+        valid_folder_keys = self._aliases.get_model_subfolder_names(model_location)
+        valid_attr_infos = self._aliases.get_model_attribute_names_and_types(model_location)
+
+        for key, value in model_node.iteritems():
+            if key in valid_folder_keys:
+                new_location = LocationContext(model_location).append_location(key)
+                self.traverse_folder(value, new_location)
+
+            elif key in valid_attr_infos:
+                self.traverse_attribute(model_node, key, model_location)
+
+            else:
+                self.unrecognized_field(model_node, key, model_location)
+
+    def traverse_attribute(self, model_dict, attribute_name, model_location):
+        """
+        Called for each attribute that is encountered in the model.
+        """
+        pass
+
+    def unrecognized_field(self, model_dict, key, model_location):
+        """
+        Called if a key in a model dictionary is not recognized as an attribute or a sub-folder.
+        This can happen if:
+        - the key is the name of a custom security provider, usually a full Java class name
+        - the key is an attribute that is not applicable for the current WLS version or offline/online status
+        - the field is not a valid attribute or folder name
+        """
+        pass

core/src/main/python/wlsdeploy/tool/util/filters/wko_filter.py

@@ -6,30 +6,64 @@
 # ------------
 # WDT filters to prepare a model for use with WKO, using the createDomain or prepareModel tools.
 # These operations can be invoked as a single call, or independently of each other.
+from wlsdeploy.aliases.validation_codes import ValidationCodes
+from wlsdeploy.aliases.wlst_modes import WlstModes
+from wlsdeploy.exception.expection_types import ExceptionType
+from wlsdeploy.tool.util.filters.model_traverse import ModelTraverse
 
-def filter_model(model):
+
+def filter_model(model, model_context):
     """
     Perform the following operations on the specified model:
     - Remove any online-only attributes
     - Check if servers in a cluster have different ports
     :param model: the model to be filtered
+    :param model_context: used by nested filters
     """
-    filter_online_attributes(model)
-    check_clustered_server_ports(model)
+    filter_online_attributes(model, model_context)
+    check_clustered_server_ports(model, model_context)
 
 
-def filter_online_attributes(model):
+def filter_online_attributes(model, model_context):
     """
     Remove any online-only attributes from the specified model.
     :param model: the model to be filtered
+    :param model_context: used to configure aliases
     """
-    print("\n\nFILTER ONLINE ATTRIBUTES")
+    online_filter = OnlineAttributeFilter(model_context, ExceptionType.PREPARE)
+    online_filter.traverse_model(model)
 
 
-def check_clustered_server_ports(model):
+def check_clustered_server_ports(model, model_context):
     """
     Set the CalculatedListenPorts attribute to false for dynamic clusters in the specified model.
     Warn if servers in a static cluster have different ports in the specified model.
     :param model: the model to be filtered
+    :param model_context: unused, passed by filter_helper if called independently
+    """
+    print("CHECK_CLUSTERED SERVER PORTS")
+
+
+class OnlineAttributeFilter(ModelTraverse):
     """
-    print("\n\nCHECK_CLUSTERED SERVER PORTS")
+    Traverse the model and remove any online-only attributes.
+    """
+    _class_name = 'OnlineAttributeFilter'
+
+    def __init__(self, model_context, exception_type):
+        # use OFFLINE regardless of tool configuration
+        ModelTraverse.__init__(self, model_context, WlstModes.OFFLINE, exception_type)
+
+    # Override
+    def unrecognized_field(self, model_dict, key, model_location):
+        """
+        If the attribute name has status ValidationCodes.VERSION_INVALID, it is a valid attribute sometimes,
+        but not for offline mode in this WLS version.
+        """
+        _method_name = 'unrecognized_field'
+
+        result, message = self._aliases.is_valid_model_attribute_name(model_location, key)
+        if result == ValidationCodes.VERSION_INVALID:
+            path = self._aliases.get_model_folder_path(model_location)
+            self._logger.info('WLSDPLY-20033', key, path, class_name=self._class_name, method_name=_method_name)
+            del model_dict[key]

core/src/main/python/wlsdeploy/util/cla_helper.py

@@ -243,7 +243,7 @@ def load_model(program_name, model_context, aliases, filter_type, wlst_mode):
         clean_up_temp_files()
         tool_exit.end(model_context, CommandLineArgUtil.PROG_ERROR_EXIT_CODE)
 
-    filter_helper.apply_filters(model_dictionary, filter_type)
+    filter_helper.apply_filters(model_dictionary, filter_type, model_context)
 
     persist_model(model_context, model_dictionary)