Add target configuration credential output method (#902)

* make sure json secrets method also injects values

* remove duplicate env variable

* refactor and introduce new field in target.json for secrets output method

* refactor and cleanup dictionary before writing to json file

* prevent model named the same as output file

* remove <user> for weblogic credentials

* do not complain existing secrets

* write new variable file to output dir if one is provided for target operation

* append the properties to the end of original properties file

* prevent complaints for @@env if target

* various fixes for handling variable files and output dir,  added handling for existing secrets but comment out

* use target configuration validation method to check for whether reporting error

* change the output file to use existing variable file name and merge/replace existing properties

* empty username in secret if it is started with @@secret:

* fixing broken method used to generate script_hash

* change file names

* fix model file names comparison with additional_output logic

Author: jshum2479

core/src/main/python/discover.py

@@ -445,10 +445,13 @@ def __check_and_customize_model(model, model_context, aliases, credential_inject
         credential_cache = credential_injector.get_variable_cache()
 
         # Generate k8s create secret script
-        if target_configuration.uses_credential_secrets():
+        if target_configuration.generate_script_for_secrets():
             target_configuration_helper.generate_k8s_script(model_context, credential_cache, model.get_model(),
                                                             ExceptionType.DISCOVER)
 
+        if target_configuration.generate_json_for_secrets():
+            target_configuration_helper.generate_k8s_json(model_context, credential_cache, model.get_model())
+
         # create additional output after filtering, but before variables have been inserted
         if model_context.is_targetted_config():
             target_configuration_helper.create_additional_output(model, model_context, aliases, credential_injector,

core/src/main/python/prepare_model.py

@@ -241,13 +241,28 @@ def walk(self):
 
         # create a merged model that is not substituted
         merged_model_dictionary = {}
-
         try:
             model_file_list = self.model_files.split(',')
+            target = self.model_context.get_target()
+
             for model_file in model_file_list:
                 if os.path.splitext(model_file)[1].lower() == ".yaml":
                     model_file_name = model_file
-                    FileToPython(model_file_name, True).parse()
+                    
+                if target is not None and self.model_context.get_target_configuration().get_additional_output_types():
+                    additional_output_types = []
+                    output_types = self.model_context.get_target_configuration().get_additional_output_types()
+                    if isinstance(output_types, list):
+                        additional_output_types.extend(output_types)
+                    else:
+                        additional_output_types.append(output_types)
+
+                    if os.path.basename(model_file_name) in additional_output_types:
+                        self._logger.severe('WLSDPLY-05802', os.path.basename(model_file_name),
+                                            additional_output_types, target)
+                        return VALIDATION_FAIL
+
+                FileToPython(model_file_name, True).parse()
 
                 aliases = Aliases(model_context=self.model_context, wlst_mode=WlstModes.OFFLINE)
 
@@ -257,7 +272,7 @@ def walk(self):
                 model_dictionary = cla_helper.merge_model_files(model_file_name, None)
 
                 variable_file = self.model_context.get_variable_file()
-                if not os.path.exists(variable_file):
+                if variable_file is not None and not os.path.exists(variable_file):
                     variable_file = None
 
                 return_code = validator.validate_in_tool_mode(model_dictionary,
@@ -299,11 +314,16 @@ def walk(self):
                                                           "discover", WlstModes.OFFLINE)
 
             target_config = self.model_context.get_target_configuration()
-            if target_config.uses_credential_secrets():
+            if target_config.generate_script_for_secrets():
                 target_configuration_helper.generate_k8s_script(self.model_context,
                                                                 self.credential_injector.get_variable_cache(),
                                                                 full_model_dictionary, ExceptionType.VALIDATE)
 
+            if target_config.generate_json_for_secrets():
+                target_configuration_helper.generate_k8s_json(self.model_context,
+                                                                self.credential_injector.get_variable_cache(),
+                                                                full_model_dictionary)
+
             # create any additional outputs from full model dictionary
             target_configuration_helper.create_additional_output(Model(full_model_dictionary), self.model_context,
                                                                  self._aliases, self.credential_injector,

core/src/main/python/wlsdeploy/json/json_translator.py

@@ -200,8 +200,12 @@ def _write_list_to_json_file(self, alist, writer, indent=''):
         for value in alist:
             writer.write(end_line)
             writer.println()
-            writer.write(list_indent)
-            writer.write(_format_json_value(value))
+            if isinstance(value, dict):
+                writer.write(list_indent)
+                self._write_dictionary_to_json_file(value, writer, indent)
+            else:
+                writer.write(list_indent)
+                writer.write(_format_json_value(value))
             end_line = ','
         writer.println()
         writer.write(indent + ']')

core/src/main/python/wlsdeploy/tool/util/targets/additional_output_helper.py

@@ -67,10 +67,9 @@ def create_additional_output(model, model_context, aliases, credential_injector,
 
     # all current output types use this hash, and process a set of template files
     template_hash = _build_template_hash(model, model_context, aliases, credential_injector)
-
-    file_names = model_context.get_target_configuration().get_additional_output_types()
-    for file_name in file_names:
-        _create_file(file_name, template_hash, model_context, output_dir, exception_type)
+    template_names = model_context.get_target_configuration().get_additional_output_types()
+    for template_name in template_names:
+        _create_file(template_name, template_hash, model_context, output_dir, exception_type)
 
 
 def _create_file(template_name, template_hash, model_context, output_dir, exception_type):

core/src/main/python/wlsdeploy/tool/util/variable_injector.py

@@ -6,8 +6,11 @@
 import re
 
 import java.lang.Boolean as Boolean
+import java.io.FileOutputStream as FileOutputStream
+import java.io.FileInputStream as FileInputStream
+import java.util.Properties as Properties
 import java.lang.IllegalArgumentException as IllegalArgumentException
-import os
+import os, shutil
 
 import oracle.weblogic.deploy.aliases.AliasException as AliasException
 import oracle.weblogic.deploy.json.JsonException as JsonException
@@ -243,6 +246,18 @@ def inject_variables_keyword_file(self, append_option=None):
 
             variable_dictionary = self.get_variable_cache()
             if variable_dictionary is not None and len(variable_dictionary) > 0:
+                # change variable_file_location to output_dir for target operation
+                if self.__model_context.get_target() is not None:
+                    new_variable_file_location = os.path.join(self.__model_context.get_output_dir(),
+                                                              os.path.basename(variable_file_location))
+                    if variable_file_location is not None and os.path.exists(variable_file_location):
+                        # copy the original file first
+                        append = True
+                        shutil.copyfile(variable_file_location, new_variable_file_location)
+                        self._filter_duplicate_properties(new_variable_file_location, variable_dictionary)
+
+                    variable_file_location = new_variable_file_location
+
                 variables_inserted = self._write_variables_file(variable_dictionary, variable_file_location, append)
             if variables_inserted:
                 _logger.info('WLSDPLY-19518', variable_file_location, class_name=_class_name,
@@ -255,6 +270,28 @@ def inject_variables_keyword_file(self, append_option=None):
         _logger.exiting(class_name=_class_name, method_name=_method_name, result=variables_inserted)
         return variables_inserted, return_model, variable_file_location
 
+
+    def _filter_duplicate_properties(self, variable_file_location, variable_dictionary):
+        _method_name = '_filter_duplicate_property'
+        _logger.entering(class_name=_class_name, method_name=_method_name)
+        try:
+            fis = FileInputStream(variable_file_location)
+            prop = Properties()
+            prop.load(fis)
+            fis.close()
+            for key in variable_dictionary:
+                if prop.get(key) is not None:
+                    prop.remove(key)
+            fos = FileOutputStream(variable_file_location)
+            prop.store(fos, None)
+            fos.close()
+        except IOException, e:
+            _logger.warning('WLSDPLY-05803', e.getLocalizedMessage(),
+                            class_name=_class_name, method_name=_method_name)
+
+        _logger.exiting(class_name=_class_name, method_name=_method_name)
+
+
     def inject_variables_keyword_dictionary(self, injector_file_list):
         """
         Takes a variable keyword dictionary and returns a variables for file in a dictionary

core/src/main/python/wlsdeploy/util/target_configuration.py

@@ -7,6 +7,7 @@
 
 # types for credential method
 CREDENTIALS_METHOD = "credentials_method"
+CREDENTIALS_OUTPUT_METHOD = "credentials_output_method"
 
 # Overrides the Kubernetes secret name for the WebLogic admin user credential
 WLS_CREDENTIALS_NAME = "wls_credentials_name"
@@ -46,6 +47,13 @@ def get_credentials_method(self):
         """
         return dictionary_utils.get_element(self.config_dictionary, CREDENTIALS_METHOD)
 
+    def get_credentials_output_method(self):
+        """
+        Returns the method for generating secrets creation method.
+        :return: script or json
+        """
+        return dictionary_utils.get_element(self.config_dictionary, CREDENTIALS_OUTPUT_METHOD)
+
     def get_wls_credentials_name(self):
         """
         Returns the method for handling credentials in the model.
@@ -101,6 +109,21 @@ def uses_credential_secrets(self):
         """
         return self.get_credentials_method() in [SECRETS_METHOD, CONFIG_OVERRIDES_SECRETS_METHOD]
 
+    def generate_script_for_secrets(self):
+        """
+        Determine if it needs to generate shell script for creating secrets.
+        :return: True if it is not equal to json
+        """
+        return not self.get_credentials_output_method() in ['json']
+
+
+    def generate_json_for_secrets(self):
+        """
+        Determine if it needs to generate json file for creating secrets.
+        :return: True if generating json file, False otherwise
+        """
+        return self.get_credentials_output_method() in ['json']
+
     def manages_credentials(self):
         """
         Determine if this configuration manages credential values in the model.

core/src/main/python/wlsdeploy/util/target_configuration_helper.py

@@ -22,6 +22,8 @@
 from wlsdeploy.tool.util.targets import file_template_helper
 from wlsdeploy.util import dictionary_utils
 from wlsdeploy.util.cla_utils import CommandLineArgUtil
+from wlsdeploy.json.json_translator import PythonToJson
+
 
 __class_name = 'target_configuration_helper'
 __logger = PlatformLogger('wlsdeploy.tool.util')
@@ -66,6 +68,7 @@
 SECURITY_NM_REPLACEMENT = 'SecurityConfig.NodeManager.'
 
 K8S_SCRIPT_NAME = 'create_k8s_secrets.sh'
+K8S_SECRET_JSON_NAME = 'k8s_secrets.json'
 K8S_SCRIPT_RESOURCE_PATH = 'oracle/weblogic/deploy/k8s/' + K8S_SCRIPT_NAME
 
 
@@ -88,20 +91,8 @@ def process_target_arguments(argument_map):
             __logger.throwing(ex, class_name=__class_name, method_name=_method_name)
             raise ex
 
-        # Set the -variable_file parameter if not present with default
-        if CommandLineArgUtil.VARIABLE_FILE_SWITCH not in argument_map:
-            path = os.path.join(output_dir, target_name + "_variable.properties")
-            argument_map[CommandLineArgUtil.VARIABLE_FILE_SWITCH] = path
 
-
-def generate_k8s_script(model_context, token_dictionary, model_dictionary, exception_type):
-    """
-    Generate a shell script for creating k8s secrets.
-    :param model_context: used to determine output directory
-    :param token_dictionary: contains every token
-    :param model_dictionary: used to determine domain UID
-    :param exception_type: type of exception to throw
-    """
+def _prepare_k8s_secrets(model_context, token_dictionary, model_dictionary):
 
     # determine the domain name and UID
     topology = dictionary_utils.get_dictionary_element(model_dictionary, TOPOLOGY)
@@ -165,13 +156,75 @@ def generate_k8s_script(model_context, token_dictionary, model_dictionary, excep
         {'text': exception_helper.get_message('WLSDPLY-01670')}
     ]
     script_hash['longMessageDetails'] = long_messages
+    return script_hash
 
+
+def generate_k8s_script(model_context, token_dictionary, model_dictionary, exception_type):
+    """
+    Generate a shell script for creating k8s secrets.
+    :param model_context: used to determine output directory
+    :param token_dictionary: contains every token
+    :param model_dictionary: used to determine domain UID
+    :param exception_type: type of exception to throw
+    """
+    script_hash = _prepare_k8s_secrets(model_context, token_dictionary, model_dictionary)
     file_location = model_context.get_output_dir()
     k8s_file = File(file_location, K8S_SCRIPT_NAME)
     file_template_helper.create_file_from_resource(K8S_SCRIPT_RESOURCE_PATH, script_hash, k8s_file, exception_type)
     FileUtils.chmod(k8s_file.getPath(), 0750)
 
 
+def generate_k8s_json(model_context, token_dictionary, model_dictionary):
+    """
+    Generate a json file.
+    :param model_context: used to determine output directory
+    :param token_dictionary: contains every token
+    :param model_dictionary: used to determine domain UID
+    :param exception_type: type of exception to throw
+    """
+    script_hash = _prepare_k8s_secrets(model_context, token_dictionary, model_dictionary)
+
+    file_location = model_context.get_output_dir()
+    k8s_file = os.path.join(file_location, K8S_SECRET_JSON_NAME)
+    result = _build_json_secrets_result(script_hash)
+    json_object = PythonToJson(result)
+    json_object.write_to_json_file(k8s_file)
+
+
+def _build_json_secrets_result(script_hash):
+
+    result = {}
+    secrets_array = []
+
+    for node in script_hash['secrets']:
+        secret = {}
+        for item in ['secretName', 'comments']:
+            secret[item] = node[item]
+        secret['keys'] = {}
+        secret['keys']['password'] = ""
+        secrets_array.append(secret)
+
+    for node in script_hash['pairedSecrets']:
+        secret = {}
+        for item in ['secretName', 'comments']:
+            secret[item] = node[item]
+        secret['keys'] = {}
+        secret['keys']['password'] = ""
+        secret['keys']['username'] = node['user']
+        # For ui, empty it now.
+        if secret['keys']['username'].startswith('@@SECRET:'):
+            secret['keys']['username'] = ""
+        if secret['secretName'] == 'weblogic-credentials':
+            secret['keys']['username'] = ""
+        secrets_array.append(secret)
+
+    result['secrets'] = secrets_array
+    result['domainUID'] = script_hash['domainUid']
+    result['namespace'] = script_hash['namespace']
+
+    return result
+
+
 def format_as_secret_token(secret_id, target_config):
     """
     Format the secret identifier as an @@SECRET token for use in a model.

core/src/main/python/wlsdeploy/util/variables.py

@@ -204,6 +204,10 @@ def _substitute(text, variables, model_context, attribute_name=None):
     :return: the replaced text
     """
     method_name = '_substitute'
+    validation_method = model_context.get_validation_method()
+    target_configuration = model_context.get_target_configuration()
+    if target_configuration:
+        validation_method = target_configuration.get_validation_method()
 
     # skip lookups for text with no @@
     if '@@' in text:
@@ -213,7 +217,8 @@ def _substitute(text, variables, model_context, attribute_name=None):
         for token, key in matches:
             # log, or throw an exception if key is not found.
             if key not in variables:
-                _report_token_issue('WLSDPLY-01732', method_name, model_context, key)
+                if validation_method != 'lax':
+                    _report_token_issue('WLSDPLY-01732', method_name, model_context, key)
                 continue
 
             value = variables[key]
@@ -224,7 +229,8 @@ def _substitute(text, variables, model_context, attribute_name=None):
         for token, key in matches:
             # log, or throw an exception if key is not found.
             if not os.environ.has_key(key):
-                _report_token_issue('WLSDPLY-01737', method_name, model_context, key)
+                if validation_method != 'lax':
+                    _report_token_issue('WLSDPLY-01737', method_name, model_context, key)
                 continue
 
             value = os.environ.get(key)
@@ -234,10 +240,13 @@ def _substitute(text, variables, model_context, attribute_name=None):
         matches = _secret_pattern.findall(text)
         for token, name, key in matches:
             value = _resolve_secret_token(name, key, model_context)
+
             if value is None:
-                secret_token = name + ':' + key
-                known_tokens = _list_known_secret_tokens()
-                _report_token_issue('WLSDPLY-01739', method_name, model_context, secret_token, known_tokens)
+                # does not match, only report for non target case
+                if validation_method != 'lax':
+                    secret_token = name + ':' + key
+                    known_tokens = _list_known_secret_tokens()
+                    _report_token_issue('WLSDPLY-01739', method_name, model_context, secret_token, known_tokens)
                 continue
 
             text = text.replace(token, value)
@@ -260,7 +269,7 @@ def _substitute(text, variables, model_context, attribute_name=None):
 
         # if any @@TOKEN: remains in the value, throw an exception
         matches = _unresolved_token_pattern.findall(text)
-        if matches:
+        if matches and validation_method != 'lax':
             match = matches[0]
             token = match[1]
             sample = "@@" + token + ":<name>"

core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties

@@ -506,6 +506,8 @@ WLSDPLY-05716=The Security Configuration Provider at location {0} has a differen
 
 # prepare_model.py
 WLSDPLY-05801=Error in prepare model {0}
+WLSDPLY-05802=Model file name {0} must not be named as one of these names: {1} when -target {2} is specified. 
+WLSDPLY-05803=Failed to filter original variable file {0}, variable file in output directory may have duplicates.
 
 ###############################################################################
 #   Message number 06000 - 07999 Discover                                     #

core/src/main/targetconfigs/k8s/target.json

@@ -6,5 +6,6 @@
       },
   "variable_injectors" : {"PORT": {},"HOST": {},"URL": {}},
   "validation_method" : "lax",
-  "credentials_method" : "secrets"
+  "credentials_method" : "secrets",
+  "credentials_output_method" : "script"
 }