Filtering on validate (#852)

* Add filtering to model validation and persist post filtering changes

* Add unit tests for filtering on validateModel and persistence post-filtering

* Apply variable substitution if variable map file defined

* Invoke filter after validation of merged model

Author: lennyphan

core/src/main/python/validate.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2017, 2020, Oracle Corporation and/or its affiliates.
+Copyright (c) 2017, 2021, Oracle Corporation and/or its affiliates.
 Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
 The WLS Deploy tooling entry point for the validateModel tool.
@@ -22,6 +22,7 @@
 from wlsdeploy.aliases.wlst_modes import WlstModes
 from wlsdeploy.exception import exception_helper
 from wlsdeploy.logging.platform_logger import PlatformLogger
+from wlsdeploy.tool.util import filter_helper
 from wlsdeploy.tool.util import model_context_helper
 from wlsdeploy.tool.validate.validator import Validator
 from wlsdeploy.util import cla_helper
@@ -117,6 +118,18 @@ def __perform_model_file_validation(model_file_name, model_context):
 
         model_validator.validate_in_standalone_mode(model_dictionary, variable_map,
                                                     model_context.get_archive_file_name())
+
+        # substitute variables before filtering
+        variables.substitute(model_dictionary, variable_map, model_context)
+        # apply filters to merged model
+        if filter_helper.apply_filters(model_dictionary, "validate"):
+            # persist model after filtering
+            cla_helper.persist_model(model_context, model_dictionary)
+
+            # validate model changes after filtering
+            model_validator.validate_in_standalone_mode(model_dictionary, variable_map,
+                                                        model_context.get_archive_file_name())
+
     except (TranslateException, VariableException), te:
         __logger.severe('WLSDPLY-20009', _program_name, model_file_name, te.getLocalizedMessage(),
                         error=te, class_name=_class_name, method_name=_method_name)

core/src/main/python/wlsdeploy/util/cla_helper.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2019, 2020, Oracle Corporation and/or its affiliates.
+Copyright (c) 2019, 2021, Oracle Corporation and/or its affiliates.
 Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
 Utility CLS methods shared by multiple tools.
@@ -243,14 +243,12 @@ def load_model(program_name, model_context, aliases, filter_type, wlst_mode):
         clean_up_temp_files()
         tool_exit.end(model_context, CommandLineArgUtil.PROG_ERROR_EXIT_CODE)
 
+    filter_helper.apply_filters(model_dictionary, filter_type)
+
     persist_model(model_context, model_dictionary)
 
     validate_model(program_name, model_dictionary, model_context, aliases, wlst_mode)
 
-    if filter_helper.apply_filters(model_dictionary, filter_type):
-        # if any filters were applied, re-validate the model
-        validate_model(program_name, model_dictionary, model_context, aliases, wlst_mode)
-
     return model_dictionary
 
 

core/src/test/python/validation_test.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2017, 2020, Oracle Corporation and/or its affiliates.
+Copyright (c) 2017, 2021, Oracle Corporation and/or its affiliates.
 Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 """
 import unittest
@@ -18,7 +18,10 @@
 from wlsdeploy.aliases.wlst_modes import WlstModes
 from wlsdeploy.aliases import alias_constants
 
+from validate import __perform_model_file_validation
+
 import oracle.weblogic.deploy.util.TranslateException as TranslateException
+import oracle.weblogic.deploy.validate.ValidateException as ValidateException
 
 from wlsdeploy.tool.create import wlsroles_helper
 
@@ -27,6 +30,8 @@ class ValidationTestCase(unittest.TestCase):
     _program_name = 'validation_test'
     _class_name = 'ValidationTestCase'
     _resources_dir = '../../test-classes'
+    # Model persistence file
+    _wlsdeply_store_model = os.path.abspath(os.getcwd()) + '/' + _resources_dir + '/validate-mii-model.json'
     # _variable_file = _resources_dir + "/test_sub_variable_file.properties"
     # _model_file = _resources_dir + '/test_empty.json'
     # _variable_file = _resources_dir + "/test_invalid_variable_file.properties"
@@ -41,10 +46,21 @@ def setUp(self):
         self._summary_handler = SummaryHandler()
         self._logger.logger.addHandler(self._summary_handler)
 
+        # Define custom configuration path for WDT
+        os.environ['WDT_CUSTOM_CONFIG'] = self._resources_dir
+        # Indicate that WDT should persist model file
+        os.environ['__WLSDEPLOY_STORE_MODEL__'] = self._wlsdeply_store_model
+
     def tearDown(self):
         # remove summary handler for next test suite
         self._logger.logger.removeHandler(self._summary_handler)
 
+        # Clean up temporary WDT custom configuration environment variables
+        # and model persistence files
+        del os.environ['WDT_CUSTOM_CONFIG']
+        del os.environ['__WLSDEPLOY_STORE_MODEL__']
+        self.deleteFile(self._wlsdeply_store_model)
+
     def testModelValidation(self):
         _method_name = 'testModelValidation'
 
@@ -161,6 +177,44 @@ def testWLSRolesValidation(self):
         self.assertEqual(handler.getMessageCount(Level.SEVERE), 0)
         self.assertEqual(handler.getMessageCount(Level.WARNING), 3)
 
+    def testFilterInvokedOnModelValidation(self):
+        """
+        Verify filter was run and changes are persisted to model file
+        """
+
+        # Setup model context arguments
+        _model_file = self._resources_dir + '/simple-model.yaml'
+        _archive_file = self._resources_dir + "/SingleAppDomain.zip"
+        _method_name = 'testFilterInvokedOnModelValidation'
+
+        mw_home = os.environ['MW_HOME']
+
+        args_map = {
+          '-oracle_home': mw_home,
+          '-model_file': _model_file,
+          '-archive_file': _archive_file
+        }
+
+        model_context = ModelContext('validate', args_map)
+
+        try:
+          # Invoke model validation
+          __perform_model_file_validation(_model_file, model_context)
+
+          # read persisted model file and convert to python dictionary
+          model_dictionary = FileToPython(self._wlsdeply_store_model, True)._parse_json()
+        except ValidateException, ve:
+          self._logger.severe('WLSDPLY-20000', self._program_name, ve.getLocalizedMessage(), error=ve,
+                        class_name=self._class_name, method_name=_method_name)
+
+        # assert the validate filter made modications and was persisted
+        self.assertEquals('gumby1234', model_dictionary['domainInfo']['AdminPassword'], "Expected validate filter to have changed AdminPassword to 'gumby1234'")
+
+    def deleteFile(self, path):
+      try:
+        os.remove(path)
+      except OSError:
+        pass
 
 if __name__ == '__main__':
     unittest.main()

core/src/test/python/wlsdeploy/util/cla_helper_test.py

@@ -1,15 +1,37 @@
 """
-Copyright (c) 2019, 2020, Oracle and/or its affiliates.
+Copyright (c) 2019, 2021, Oracle and/or its affiliates.
 Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl.
 """
+import os
 import unittest
 
+from wlsdeploy.aliases.aliases import Aliases
+from wlsdeploy.aliases.wlst_modes import WlstModes
+from wlsdeploy.exception.expection_types import ExceptionType
+from wlsdeploy.util.model_context import ModelContext
 from wlsdeploy.util import cla_helper
 
 
 class ClaHelperTest(unittest.TestCase):
 
+    _resources_dir = '../../test-classes'
+    # Model persistence file
+    _wlsdeply_store_model = os.path.abspath(os.getcwd()) + '/' + _resources_dir + '/validate-mii-model.json'
     properties_file = '../../test-classes/test.properties'
+
+    def setUp(self):
+        # Define custom configuration path for WDT
+        os.environ['WDT_CUSTOM_CONFIG'] = self._resources_dir
+        # Indicate that WDT should persist model file
+        os.environ['__WLSDEPLOY_STORE_MODEL__'] = self._wlsdeply_store_model
+
+    def tearDown(self):
+        # Clean up temporary WDT custom configuration environment variables
+        # and model persistence files
+        del os.environ['WDT_CUSTOM_CONFIG']
+        del os.environ['__WLSDEPLOY_STORE_MODEL__']
+        deleteFile(self._wlsdeply_store_model)
+
     # merging should combine elements with the same name (m1), add any elements only in the second model (m3),
     # and leave existing elements (m2) in place.
     def testMergeModels(self):
@@ -136,6 +158,33 @@ def testMergeNameToDeletedName(self):
         server = self._check_single_server(dictionary, 'm1')
         self.assertEquals(2, len(server), "server should have two attributes")
 
+    def testPersistModelAfterFilter(self):
+        """
+        Verify filter was run and changes are persisted to model file
+        """
+        # Setup model context arguments
+        _model_file = self._resources_dir + '/simple-model.yaml'
+        _archive_file = self._resources_dir + "/SingleAppDomain.zip"
+        _method_name = 'testPersistModelAfterFilter'
+
+        mw_home = os.environ['MW_HOME']
+
+        args_map = {
+            '-oracle_home': mw_home,
+            '-model_file': _model_file,
+            '-archive_file': _archive_file
+        }
+
+        model_context = ModelContext('validate', args_map)
+
+        aliases = Aliases(model_context, wlst_mode=WlstModes.OFFLINE, exception_type=ExceptionType.DEPLOY)
+
+        # Load model and invoke filter
+        model_dictionary = cla_helper.load_model('validateModel', model_context, aliases, "validate", WlstModes.OFFLINE)
+
+        # assert the validate filter made modications and was persisted
+        self.assertEquals('gumby1234', model_dictionary['domainInfo']['AdminPassword'], "Expected validate filter to have changed AdminPassword to 'gumby1234'")
+
         # check that a single server exists in the result, and its attributes were merged correctly
     def _check_merged_server(self, dictionary, key):
         server = self._check_single_server(dictionary, key)
@@ -190,3 +239,9 @@ def _build_variable_map():
         "server1a": "m1",
         "server1b": "m1"
     }
+
+def deleteFile(path):
+    try:
+        os.remove(path)
+    except OSError:
+        pass

core/src/test/resources/model_filters.json

@@ -0,0 +1,13 @@
+{
+    "create": [
+    ],
+    "deploy": [
+    ],
+    "discover": [
+    ],
+    "update": [
+    ],
+    "validate": [
+      { "name": "validate_filter", "path": "../../test-classes/wdt_validate_filter.py" }
+    ]
+  }

core/src/test/resources/wdt_validate_filter.py

@@ -0,0 +1,12 @@
+# Copyright (c) 2018, 2021, Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
+#
+# ------------
+# Description:
+# ------------
+#  This is a WDT filter for handling WLS configuration overrides.
+#
+
+def filter_model(model):
+  if 'domainInfo' in model:
+    model['domainInfo']['AdminPassword'] = 'gumby1234'