Allow multiple variable files for create, update, deploy and extract (#669)

* JIRA WDT-374 - Allow multiple variable files for some tools

* JIRA WDT-374 - Update script usage to allow multiple variable files

* JIRA WDT-374 - Corrected logging parameters

* JIRA WDT-374 - Handle case where variable file can't be written; show summary logging

* JIRA WDT-374 - Use absolute paths to build variable file list

Author: rakillen

core/src/main/python/prepare_model.py

@@ -36,6 +36,7 @@
 from wlsdeploy.exception.expection_types import ExceptionType
 from wlsdeploy.logging.platform_logger import PlatformLogger
 from wlsdeploy.tool.util import filter_helper
+from wlsdeploy.tool.util import model_context_helper
 from wlsdeploy.tool.util import variable_injector_functions
 from wlsdeploy.tool.util.alias_helper import AliasHelper
 from wlsdeploy.tool.util.variable_injector import VariableInjector
@@ -44,6 +45,7 @@
 from wlsdeploy.util import cla_helper
 from wlsdeploy.util import model
 from wlsdeploy.util import target_configuration_helper
+from wlsdeploy.util import tool_exit
 from wlsdeploy.util.cla_utils import CommandLineArgUtil
 from wlsdeploy.util.model import Model
 from wlsdeploy.util.model_context import ModelContext
@@ -455,8 +457,10 @@ def _apply_filter_and_inject_variable(self, model, model_context, validator):
                     variable_map = validator.load_variables(self.model_context.get_variable_file())
                     self.cache.update(variable_map)
 
-        variable_injector.inject_variables_keyword_file()
-        return model
+        inserted, variable_model, variable_file_name = variable_injector.inject_variables_keyword_file()
+        # return variable_model - if writing the variables file failed, this will be the original model.
+        # a warning is issued in inject_variables_keyword_file() if that was the case.
+        return variable_model
 
 def debug(format_string, *arguments):
     """
@@ -480,6 +484,8 @@ def main():
     for index, arg in enumerate(sys.argv):
         __logger.finer('sys.argv[{0}] = {1}', str(index), str(arg), class_name=_class_name, method_name=_method_name)
 
+    # create a minimal model for summary logging
+    model_context = model_context_helper.create_exit_context(_program_name)
     _outputdir = None
 
     try:
@@ -495,29 +501,29 @@ def main():
 
         obj = PrepareModel(model1, model_context, __logger, _outputdir)
         rc = obj.walk()
-        System.exit(rc)
+        tool_exit.end(model_context, rc)
 
     except CLAException, ex:
         exit_code = 2
         if exit_code != CommandLineArgUtil.HELP_EXIT_CODE:
             __logger.severe('WLSDPLY-20008', _program_name, ex.getLocalizedMessage(), error=ex,
                             class_name=_class_name, method_name=_method_name)
         cla_helper.clean_up_temp_files()
-        sys.exit(exit_code)
+        tool_exit.end(model_context, exit_code)
     except CompareException, ce:
         cla_helper.clean_up_temp_files()
         __logger.severe('WLSDPLY-05704', ce.getLocalizedMessage())
-        System.exit(2)
+        tool_exit.end(model_context, 2)
     except PyWLSTException, pe:
         cla_helper.clean_up_temp_files()
         __logger.severe('WLSDPLY-05704', pe.getLocalizedMessage())
-        System.exit(2)
+        tool_exit.end(model_context, 2)
     except:
         exc_type, exc_obj, exc_tb = sys.exc_info()
         eeString = traceback.format_exception(exc_type, exc_obj, exc_tb)
         cla_helper.clean_up_temp_files()
         __logger.severe('WLSDPLY-05704', eeString)
-        System.exit(2)
+        tool_exit.end(model_context, 2)
 
 def format_message(key, *args):
     """

core/src/main/python/wlsdeploy/util/cla_helper.py

@@ -117,19 +117,30 @@ def validate_model_present(program_name, optional_arg_map):
     return
 
 
-def validate_variable_file_exists(program_name, optional_arg_map):
-    method_name = '_validate_variable_file_arg'
-    if CommandLineArgUtil.VARIABLE_FILE_SWITCH in optional_arg_map:
-        value = optional_arg_map[CommandLineArgUtil.VARIABLE_FILE_SWITCH]
-        try:
-            variable_file = FileUtils.validateExistingFile(value)
-        except IllegalArgumentException, iae:
-            ex = exception_helper.create_cla_exception('WLSDPLY-20031', program_name, value,
-                                                       iae.getLocalizedMessage(), error=iae)
-            ex.setExitCode(CommandLineArgUtil.ARG_VALIDATION_ERROR_EXIT_CODE)
-            __logger.throwing(ex, class_name=_class_name, method_name=method_name)
-            raise ex
-        optional_arg_map[CommandLineArgUtil.VARIABLE_FILE_SWITCH] = variable_file.getAbsolutePath()
+def validate_variable_file_exists(program_name, argument_map):
+    """
+    Validate that the variable file(s) exist.
+    Assume that the caller allows multiple variables files.
+    :param program_name: the name of the tool
+    :param argument_map: the program arguments
+    """
+    method_name = 'validate_variable_file_exists'
+    if CommandLineArgUtil.VARIABLE_FILE_SWITCH in argument_map:
+        result_files = []  # type: list
+        value = argument_map[CommandLineArgUtil.VARIABLE_FILE_SWITCH]
+        files = value.split(CommandLineArgUtil.MODEL_FILES_SEPARATOR)
+        for file in files:
+            try:
+                variable_file = FileUtils.validateExistingFile(file)
+                result_files.append(variable_file.getAbsolutePath())
+            except IllegalArgumentException, iae:
+                ex = exception_helper.create_cla_exception('WLSDPLY-20031', program_name, file,
+                                                           iae.getLocalizedMessage(), error=iae)
+                ex.setExitCode(CommandLineArgUtil.ARG_VALIDATION_ERROR_EXIT_CODE)
+                __logger.throwing(ex, class_name=_class_name, method_name=method_name)
+                raise ex
+
+        argument_map[CommandLineArgUtil.VARIABLE_FILE_SWITCH] = ",".join(result_files)
     return
 
 
@@ -204,7 +215,8 @@ def load_model(program_name, model_context, aliases, filter_type, wlst_mode):
     variable_map = {}
     try:
         if model_context.get_variable_file():
-            variable_map = variables.load_variables(model_context.get_variable_file())
+            # callers of this method allow multiple variable files
+            variable_map = variables.load_variables(model_context.get_variable_file(), allow_multiple_files=True)
     except VariableException, ex:
         __logger.severe('WLSDPLY-20004', program_name, ex.getLocalizedMessage(), error=ex,
                         class_name=_class_name, method_name=_method_name)

core/src/main/python/wlsdeploy/util/variables.py

@@ -21,6 +21,7 @@
 from wlsdeploy.exception import exception_helper
 from wlsdeploy.logging import platform_logger
 from wlsdeploy.util import dictionary_utils
+from wlsdeploy.util.cla_utils import CommandLineArgUtil
 
 _class_name = "variables"
 _logger = platform_logger.PlatformLogger('wlsdeploy.variables')
@@ -39,33 +40,41 @@
 _secret_token_map = None
 
 
-def load_variables(file_path):
+def load_variables(file_path, allow_multiple_files=False):
     """
-    Load a dictionary of variables from the specified file.
+    Load a dictionary of variables from the specified file(s).
     :param file_path: the file from which to load properties
+    :param allow_multiple_files: if True, allow a comma-separated list of variable files
     :return the dictionary of variables
     :raises VariableException if an I/O error occurs while loading the variables from the file
     """
     method_name = "load_variables"
 
+    if allow_multiple_files:
+        paths = file_path.split(CommandLineArgUtil.MODEL_FILES_SEPARATOR)
+    else:
+        paths = [file_path]
+
     variable_map = {}
-    props = Properties()
-    input_stream = None
-    try:
-        input_stream = FileInputStream(file_path)
-        props.load(input_stream)
-        input_stream.close()
-    except IOException, ioe:
-        ex = exception_helper.create_variable_exception('WLSDPLY-01730', file_path,
-                                                        ioe.getLocalizedMessage(), error=ioe)
-        _logger.throwing(ex, class_name=_class_name, method_name=method_name)
-        if input_stream is not None:
-            input_stream.close()
-        raise ex
 
-    for key in props.keySet():
-        value = props.getProperty(key)
-        variable_map[key] = value
+    for path in paths:
+        props = Properties()
+        input_stream = None
+        try:
+            input_stream = FileInputStream(path)
+            props.load(input_stream)
+            input_stream.close()
+        except IOException, ioe:
+            ex = exception_helper.create_variable_exception('WLSDPLY-01730', path, ioe.getLocalizedMessage(),
+                                                            error=ioe)
+            _logger.throwing(ex, class_name=_class_name, method_name=method_name)
+            if input_stream is not None:
+                input_stream.close()
+            raise ex
+
+        for key in props.keySet():
+            value = props.getProperty(key)
+            variable_map[key] = value
 
     return variable_map
 
@@ -105,8 +114,8 @@ def write_ordered_variables(program_name, variable_map, file_path, append=False)
             pw.println(formatted)
         pw.close()
     except IOException, ioe:
-        _logger.fine('WLSDPLY-20007', file_path, ioe.getLocalizedMessage())
-        ex = exception_helper.create_variable_exception('WLSDPLY-20007', file_path,
+        _logger.fine('WLSDPLY-20007', program_name, file_path, ioe.getLocalizedMessage())
+        ex = exception_helper.create_variable_exception('WLSDPLY-20007', program_name, file_path,
                                                         ioe.getLocalizedMessage(), error=ioe)
         _logger.throwing(ex, class_name=_class_name, method_name=_method_name)
         if pw is not None:

installer/src/main/bin/createDomain.cmd

@@ -116,9 +116,9 @@ ECHO         model_file      - the location of the model file to use.  This can
 ECHO                           comma-separated list of model locations, where each successive model
 ECHO                           layers on top of the previous ones.
 ECHO.
-ECHO         variable_file   - the location of the property file containing
-ECHO                           the variable values for all variables used in
-ECHO                           the model
+ECHO         variable_file   - the location of the property file containing the values for variables used in
+ECHO                           the model. This can also be specified as a comma-separated list of property files,
+ECHO                           where each successive set of properties layers on top of the previous ones.
 ECHO.
 ECHO         wlst_path       - the Oracle Home subdirectory of the wlst.cmd
 ECHO                           script to use (e.g., ^<ORACLE_HOME^>\soa).

installer/src/main/bin/createDomain.sh

@@ -77,9 +77,9 @@ usage() {
   echo "                          comma-separated list of model locations, where each successive model layers on top"
   echo "                          of the previous ones."
   echo ""
-  echo "        variable_file   - the location of the property file containing"
-  echo "                          the variable values for all variables used in"
-  echo "                          the model"
+  echo "        variable_file   - the location of the property file containing the values for variables used in"
+  echo "                          the model. This can also be specified as a comma-separated list of property files,"
+  echo "                          where each successive set of properties layers on top of the previous ones."
   echo ""
   echo "        wlst_path       - the Oracle Home subdirectory of the wlst.cmd"
   echo "                          script to use (e.g., <ORACLE_HOME>/soa)."

installer/src/main/bin/deployApps.cmd

@@ -99,9 +99,9 @@ ECHO         model_file      - the location of the model file to use.  This can
 ECHO                           comma-separated list of model locations, where each successive model
 ECHO                           layers on top of the previous ones.
 ECHO.
-ECHO         variable_file   - the location of the property file containing
-ECHO                           the variable values for all variables used in
-ECHO                           the model
+ECHO         variable_file   - the location of the property file containing the values for variables used in
+ECHO                           the model. This can also be specified as a comma-separated list of property files,
+ECHO                           where each successive set of properties layers on top of the previous ones.
 ECHO.
 ECHO         domain_type     - the type of domain (e.g., WLS, JRF).
 ECHO                           Used to locate wlst.cmd if -wlst_path not specified

installer/src/main/bin/deployApps.sh

@@ -60,9 +60,9 @@ usage() {
   echo "                          comma-separated list of model locations, where each successive model layers"
   echo "                          on top of the previous ones."
   echo ""
-  echo "        variable_file   - the location of the property file containing"
-  echo "                          the variable values for all variables used in"
-  echo "                          the model"
+  echo "        variable_file   - the location of the property file containing the values for variables used in"
+  echo "                          the model. This can also be specified as a comma-separated list of property files,"
+  echo "                          where each successive set of properties layers on top of the previous ones."
   echo ""
   echo "        domain_type     - the type of domain (e.g., WLS, JRF)."
   echo "                          Used to locate wlst.cmd if -wlst_path not specified"

installer/src/main/bin/extractDomainResource.cmd

@@ -95,9 +95,9 @@ ECHO         model_file      - the location of the model file to use.  This can
 ECHO                           comma-separated list of model locations, where each successive model
 ECHO                           layers on top of the previous ones.
 ECHO.
-ECHO         variable_file   - the location of the property file containing
-ECHO                           the variable values for all variables used in
-ECHO                           the model
+ECHO         variable_file   - the location of the property file containing the values for variables used in
+ECHO                           the model. This can also be specified as a comma-separated list of property files,
+ECHO                           where each successive set of properties layers on top of the previous ones.
 ECHO.
 ECHO    The -use_encryption switch tells the program that one or more of the
 ECHO    passwords in the model or variables files are encrypted.  The program will

installer/src/main/bin/extractDomainResource.sh

@@ -56,9 +56,9 @@ usage() {
   echo "                          comma-separated list of model locations, where each successive model layers"
   echo "                          on top of the previous ones."
   echo ""
-  echo "        variable_file   - the location of the property file containing"
-  echo "                          the variable values for all variables used in"
-  echo "                          the model"
+  echo "        variable_file   - the location of the property file containing the values for variables used in"
+  echo "                          the model. This can also be specified as a comma-separated list of property files,"
+  echo "                          where each successive set of properties layers on top of the previous ones."
   echo ""
   echo "    The -use_encryption switch tells the program that one or more of the"
   echo "    passwords in the model or variables files are encrypted.  The program will"

installer/src/main/bin/updateDomain.cmd

@@ -99,9 +99,9 @@ ECHO         model_file      - the location of the model file to use.  This can
 ECHO                           comma-separated list of model locations, where each successive model
 ECHO                           layers on top of the previous ones.
 ECHO.
-ECHO         variable_file   - the location of the property file containing
-ECHO                           the variable values for all variables used in
-ECHO                           the model
+ECHO         variable_file   - the location of the property file containing the values for variables used in
+ECHO                           the model. This can also be specified as a comma-separated list of property files,
+ECHO                           where each successive set of properties layers on top of the previous ones.
 ECHO.
 ECHO         domain_type     - the type of domain (e.g., WLS, JRF).
 ECHO                           Used to locate wlst.cmd if -wlst_path not specified