Merge pull request #1017 from oracle/develop-filters

Avoid multiple server ports in cluster, remove online attributes for WKO

Author: robertpatrick

core/src/main/python/validate.py

@@ -122,7 +122,7 @@ def __perform_model_file_validation(model_file_name, model_context):
         # substitute variables before filtering
         variables.substitute(model_dictionary, variable_map, model_context)
         # apply filters to merged model
-        if filter_helper.apply_filters(model_dictionary, "validate"):
+        if filter_helper.apply_filters(model_dictionary, "validate", model_context):
             # persist model after filtering
             cla_helper.persist_model(model_context, model_dictionary)
 

core/src/main/python/wlsdeploy/aliases/model_constants.py

@@ -41,6 +41,7 @@
 AUDITOR = 'Auditor'
 AUTHENTICATION_PROVIDER = 'AuthenticationProvider'
 AUTHORIZER = 'Authorizer'
+CALCULATED_LISTEN_PORTS = 'CalculatedListenPorts'
 CAPACITY = 'Capacity'
 CDI_CONTAINER = 'CdiContainer'
 CERT_PATH_PROVIDER = 'CertPathProvider'

core/src/main/python/wlsdeploy/exception/exception_helper.py

@@ -40,6 +40,7 @@
     ExceptionType.DISCOVER:              'create_discover_exception',
     ExceptionType.ENCRYPTION:            'create_encryption_exception',
     ExceptionType.JSON:                  'create_json_exception',
+    ExceptionType.PREPARE:               'create_prepare_exception',
     ExceptionType.PY_WLST:               'create_pywlst_exception',
     ExceptionType.TRANSLATE:             'create_translate_exception',
     ExceptionType.VALIDATE:              'create_validate_exception',

core/src/main/python/wlsdeploy/tool/util/filter_helper.py

@@ -1,12 +1,13 @@
 """
-Copyright (c) 2017, 2020, Oracle Corporation and/or its affiliates.  All rights reserved.
+Copyright (c) 2017, 2021, Oracle Corporation and/or its affiliates.  All rights reserved.
 Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 """
 import imp
 import os
 import sys
 
 from wlsdeploy.logging.platform_logger import PlatformLogger
+from wlsdeploy.tool.util.filters import wko_filter
 from wlsdeploy.util import dictionary_utils
 from wlsdeploy.util import path_utils
 from wlsdeploy.util.model_translator import FileToPython
@@ -17,30 +18,30 @@
 TARGET_CONFIG_TOKEN = '@@TARGET_CONFIG_DIR@@'
 
 __id_filter_map = {
-    # 'filterId': filter_method
+    'wko_filter': wko_filter.filter_model
 }
 
 
-def apply_filters(model, tool_type, model_context=None):
+def apply_filters(model, tool_type, model_context):
     """
     Apply any filters configured for the specified tool type to the specified model.
     :param model: the model to be filtered
     :param tool_type: the name of the filter tool type
-    :param model_context: optional, used to find target filters
+    :param model_context: used to find target filters
     :return: True if any filter was applied, False otherwise
     :raises: BundleAwareException of the specified type: if an error occurs
     """
     _method_name = 'apply_filters'
-    global __filter_file_location
 
-    __filter_file_location = path_utils.find_config_path('model_filters.json')
+    __filter_file_location = None
     filter_applied = False
 
     try:
         filters_dictionary = {}
 
         # if target specified in model context, use the filters from target config
         if model_context and model_context.get_target():
+            __filter_file_location = model_context.get_target_configuration_file()
             filters_dictionary = model_context.get_target_configuration().get_model_filters()
             target_path = os.path.join('targets', model_context.get_target())
 
@@ -52,15 +53,18 @@ def apply_filters(model, tool_type, model_context=None):
                         filter_path = target_path + filter_path.replace(TARGET_CONFIG_TOKEN, '')
                         current_filter['path'] = path_utils.find_config_path(filter_path)
 
-        elif os.path.isfile(__filter_file_location):
-            filters_dictionary = FileToPython(__filter_file_location).parse()
         else:
-            __logger.info('WLSDPLY-20017', __filter_file_location, class_name=__class_name, method_name=_method_name)
+            __filter_file_location = path_utils.find_config_path('model_filters.json')
+            if os.path.isfile(__filter_file_location):
+                filters_dictionary = FileToPython(__filter_file_location).parse()
+            else:
+                __logger.info('WLSDPLY-20017', __filter_file_location, class_name=__class_name,
+                              method_name=_method_name)
 
         if tool_type in filters_dictionary:
             filter_list = filters_dictionary[tool_type]
             for filter in filter_list:
-                filter_applied = _apply_filter(model, filter) or filter_applied
+                filter_applied = _apply_filter(model, filter, model_context, __filter_file_location) or filter_applied
         else:
             __logger.info('WLSDPLY-20016', tool_type, __filter_file_location, class_name=__class_name,
                           method_name=_method_name)
@@ -71,34 +75,39 @@ def apply_filters(model, tool_type, model_context=None):
     return filter_applied
 
 
-def _apply_filter(model, the_filter):
+def _apply_filter(model, the_filter, model_context, filter_file_location):
     """
     Apply the specified filter to the specified model.
     :param model: the model to be filtered
     :param the_filter: a dictionary containing the filter parameters
+    :param model_context: may be used by internal (ID) filters
+    :param filter_file_location: used for logging
     :return: True if the specified filter was applied, False otherwise
     :raises: BundleAwareException of the specified type: if an error occurs
     """
     _method_name = '_apply_filter'
-    global __filter_file_location
 
     id = dictionary_utils.get_element(the_filter, 'id')
     if id is not None:
-        return _apply_id_filter(model, id)
+        __logger.info('WLSDPLY-20034', id, class_name=__class_name, method_name=_method_name)
+        return _apply_id_filter(model, id, model_context)
 
     path = dictionary_utils.get_element(the_filter, 'path')
     if path is not None:
+        name = dictionary_utils.get_element(the_filter, 'name')
+        __logger.info('WLSDPLY-20033', name, class_name=__class_name, method_name=_method_name)
         return _apply_path_filter(model, path)
 
-    __logger.severe('WLSDPLY-20019', str(__filter_file_location), class_name=__class_name, method_name=_method_name)
+    __logger.severe('WLSDPLY-20019', str(filter_file_location), class_name=__class_name, method_name=_method_name)
     return False
 
 
-def _apply_id_filter(model, id):
+def _apply_id_filter(model, id, model_context):
     """
     Apply the specified ID filter to the specified model.
     :param model: the model to be filtered
     :param id: the ID of the filter to be applied
+    :param model_context: may be used by filters
     :return: True if the specified filter was applied, False otherwise
     :raises: BundleAwareException of the specified type: if an error occurs
     """
@@ -109,7 +118,7 @@ def _apply_id_filter(model, id):
         __logger.severe('WLSDPLY-20020', str(id), class_name=__class_name, method_name=_method_name)
         return False
     else:
-        filter_method(model)
+        filter_method(model, model_context)
         return True
 
 

core/src/main/python/wlsdeploy/tool/util/filters/__init__.py

@@ -0,0 +1,4 @@
+"""
+Copyright (c) 2021, Oracle Corporation and/or its affiliates.
+Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
+"""

core/src/main/python/wlsdeploy/tool/util/filters/model_traverse.py

@@ -0,0 +1,109 @@
+# Copyright (c) 2021, Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
+
+from wlsdeploy.aliases.aliases import Aliases
+from wlsdeploy.aliases.location_context import LocationContext
+from wlsdeploy.aliases.model_constants import APP_DEPLOYMENTS
+from wlsdeploy.aliases.model_constants import DOMAIN_INFO
+from wlsdeploy.aliases.model_constants import RESOURCES
+from wlsdeploy.aliases.model_constants import TOPOLOGY
+from wlsdeploy.logging.platform_logger import PlatformLogger
+
+_class_name = 'ModelTraverse'
+
+
+class ModelTraverse:
+    """
+    This class traverses all model folders and attributes.
+    Sub-classes can override specific methods to avoid re-implementing the recursive traversal.
+    """
+    def __init__(self, model_context, wlst_mode, exception_type):
+        self._aliases = Aliases(model_context=model_context, wlst_mode=wlst_mode, exception_type=exception_type)
+        self._logger = PlatformLogger('wlsdeploy.model_traverse')
+
+    def traverse_model(self, root_dict):
+        section_names = [DOMAIN_INFO, TOPOLOGY, RESOURCES, APP_DEPLOYMENTS]
+        for section_name in section_names:
+            self.traverse_section(root_dict, section_name)
+
+    def traverse_section(self, model_dict, section_name):
+        if section_name not in model_dict.keys():
+            return
+
+        # only specific top-level sections have attributes
+        valid_attribute_names = []
+        attribute_location = self._aliases.get_model_section_attribute_location(section_name)
+        if attribute_location is not None:
+            valid_attribute_names = self._aliases.get_model_attribute_names_and_types(attribute_location)
+
+        model_location = LocationContext()
+        model_section_dict = model_dict[section_name]
+        valid_folder_names = self._aliases.get_model_section_top_level_folder_names(section_name)
+        self.traverse_node_elements(model_section_dict, model_location, valid_folder_names, valid_attribute_names)
+
+    def traverse_folder(self, model_node, model_location):
+        """
+        Traverse a folder that may have named sub-folders (such as Server), artificial named sub-folders (such
+        as a security provider type), or its own sub-folders and attributes (such as JTA).
+        """
+
+        # avoid checking folder type if is_artificial_type_folder
+        is_multiple = not self._aliases.is_artificial_type_folder(model_location) and \
+            (self._aliases.supports_multiple_mbean_instances(model_location) or
+                self._aliases.requires_artificial_type_subfolder_handling(model_location))
+
+        if is_multiple:
+            for name in model_node:
+                expanded_name = name
+                new_location = LocationContext(model_location)
+                name_token = self._aliases.get_name_token(new_location)
+                if name_token is not None:
+                    new_location.add_name_token(name_token, expanded_name)
+                value_dict = model_node[name]
+                self.traverse_node(value_dict, new_location)
+        else:
+            name_token = self._aliases.get_name_token(model_location)
+            if name_token is not None:
+                name = model_location.get_name_for_token(name_token)
+                if name is None:
+                    name = '%s-0' % name_token
+                model_location.add_name_token(name_token, name)
+            self.traverse_node(model_node, model_location)
+
+    def traverse_node(self, model_node, model_location):
+        valid_folder_names = self._aliases.get_model_subfolder_names(model_location)
+        valid_attribute_names = self._aliases.get_model_attribute_names(model_location)
+        self.traverse_node_elements(model_node, model_location, valid_folder_names, valid_attribute_names)
+
+    def traverse_node_elements(self, model_node, model_location, valid_folder_names, valid_attribute_names):
+        """
+        Traverse a node that contains only attributes, non-named sub-folders, and artificial type folders.
+        """
+
+        # use items(), not iteritems(), to avoid ConcurrentModificationException if node is modified
+        for key, value in model_node.items():
+            if key in valid_folder_names:
+                new_location = LocationContext(model_location).append_location(key)
+                self.traverse_folder(value, new_location)
+
+            elif key in valid_attribute_names:
+                self.traverse_attribute(model_node, key, model_location)
+
+            else:
+                self.unrecognized_field(model_node, key, model_location)
+
+    def traverse_attribute(self, model_dict, attribute_name, model_location):
+        """
+        Called for each attribute that is encountered in the model.
+        """
+        pass
+
+    def unrecognized_field(self, model_dict, key, model_location):
+        """
+        Called if a key in a model dictionary is not recognized as an attribute or a sub-folder.
+        This can happen if:
+        - the key is the name of a custom security provider, usually a full Java class name
+        - the key is an attribute that is not applicable for the current WLS version or offline/online status
+        - the field is not a valid attribute or folder name
+        """
+        pass

core/src/main/python/wlsdeploy/tool/util/filters/wko_filter.py

@@ -0,0 +1,121 @@
+# Copyright (c) 2021, Oracle Corporation and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
+#
+# ------------
+# Description:
+# ------------
+# WDT filters to prepare a model for use with WKO, using the createDomain or prepareModel tools.
+# These operations can be invoked as a single call, or independently of each other.
+
+from wlsdeploy.aliases import alias_utils
+from wlsdeploy.aliases.model_constants import CALCULATED_LISTEN_PORTS
+from wlsdeploy.aliases.model_constants import CLUSTER
+from wlsdeploy.aliases.model_constants import DYNAMIC_SERVERS
+from wlsdeploy.aliases.model_constants import LISTEN_PORT
+from wlsdeploy.aliases.model_constants import SERVER
+from wlsdeploy.aliases.model_constants import TOPOLOGY
+from wlsdeploy.aliases.validation_codes import ValidationCodes
+from wlsdeploy.aliases.wlst_modes import WlstModes
+from wlsdeploy.exception.expection_types import ExceptionType
+from wlsdeploy.logging.platform_logger import PlatformLogger
+from wlsdeploy.tool.util.filters.model_traverse import ModelTraverse
+from wlsdeploy.util import dictionary_utils
+
+_class_name = 'wko_filter'
+_logger = PlatformLogger('wlsdeploy.tool.util')
+
+
+def filter_model(model, model_context):
+    """
+    Perform the following operations on the specified model:
+    - Remove any online-only attributes
+    - Check if servers in a cluster have different ports
+    :param model: the model to be filtered
+    :param model_context: used by nested filters
+    """
+    filter_online_attributes(model, model_context)
+    check_clustered_server_ports(model, model_context)
+
+
+def filter_online_attributes(model, model_context):
+    """
+    Remove any online-only attributes from the specified model.
+    :param model: the model to be filtered
+    :param model_context: used to configure aliases
+    """
+    online_filter = OnlineAttributeFilter(model_context, ExceptionType.PREPARE)
+    online_filter.traverse_model(model)
+
+
+def check_clustered_server_ports(model, _model_context):
+    """
+    Set the CalculatedListenPorts attribute to false for dynamic clusters in the specified model.
+    Warn if servers in a static cluster have different ports in the specified model.
+    :param model: the model to be filtered
+    :param _model_context: unused, passed by filter_helper if called independently
+    """
+    _method_name = 'check_clustered_server_ports'
+
+    topology_folder = dictionary_utils.get_dictionary_element(model, TOPOLOGY)
+
+    # be sure every dynamic cluster has DynamicServers/CalculatedListenPorts set to false
+
+    clusters_folder = dictionary_utils.get_dictionary_element(topology_folder, CLUSTER)
+    for cluster_name, cluster_fields in clusters_folder.items():
+        dynamic_folder = cluster_fields[DYNAMIC_SERVERS]
+        if dynamic_folder:
+            calculated_listen_ports = dynamic_folder[CALCULATED_LISTEN_PORTS]
+            if (calculated_listen_ports is None) or alias_utils.convert_boolean(calculated_listen_ports):
+                _logger.info('WLSDPLY-20202', CALCULATED_LISTEN_PORTS, CLUSTER, cluster_name, class_name=_class_name,
+                             method_name=_method_name)
+                dynamic_folder[CALCULATED_LISTEN_PORTS] = False
+
+    # be sure every server assigned to a cluster has the same listen port
+
+    server_port_map = {}
+    servers_folder = dictionary_utils.get_dictionary_element(topology_folder, SERVER)
+    for server_name, server_fields in servers_folder.items():
+        server_cluster = server_fields[CLUSTER]
+        server_port = server_fields[LISTEN_PORT]
+
+        if server_cluster and (server_port is not None):
+            server_port_text = str(server_port)
+            if '@@' in server_port_text:
+                # prepareModel filters the model before and after it is tokenized,
+                # so disregard variable values in the tokenized pass
+                continue
+
+            if server_cluster in server_port_map:
+                cluster_info = server_port_map[server_cluster]
+                first_server = cluster_info["firstServer"]
+                cluster_port = cluster_info["serverPort"]
+                if server_port_text != cluster_port:
+                    _logger.warning('WLSDPLY-20203', SERVER, first_server, server_name, CLUSTER, server_cluster,
+                                    LISTEN_PORT, cluster_port, server_port_text, class_name=_class_name,
+                                    method_name=_method_name)
+            else:
+                server_port_map[server_cluster] = {"firstServer": server_name, "serverPort": server_port_text}
+
+
+class OnlineAttributeFilter(ModelTraverse):
+    """
+    Traverse the model and remove any online-only attributes.
+    """
+
+    def __init__(self, model_context, exception_type):
+        # use OFFLINE regardless of tool configuration
+        ModelTraverse.__init__(self, model_context, WlstModes.OFFLINE, exception_type)
+
+    # Override
+    def unrecognized_field(self, model_dict, key, model_location):
+        """
+        If the attribute name has status ValidationCodes.VERSION_INVALID, it is a valid attribute sometimes,
+        but not for offline mode in this WLS version.
+        """
+        _method_name = 'unrecognized_field'
+
+        result, message = self._aliases.is_valid_model_attribute_name(model_location, key)
+        if result == ValidationCodes.VERSION_INVALID:
+            path = self._aliases.get_model_folder_path(model_location)
+            _logger.info('WLSDPLY-20201', key, path, class_name=_class_name, method_name=_method_name)
+            del model_dict[key]