Merge branch 'wdt-850-851' into 'develop-4.0'

Fixing -remote for structured apps and discover issue with tokens

See merge request weblogic-cloud/weblogic-deploy-tooling!1643

Author: robertpatrick

core/src/main/python/wlsdeploy/tool/deploy/applications_online_deployer.py

@@ -996,9 +996,16 @@ def __get_deploy_options(self, model_apps, app_name, is_library_module):
             deploy_options['libraryModule'] = 'true'
 
         if self.model_context.is_remote():
-            deploy_options['upload'] = 'true'
-
-        if self.model_context.is_ssh():
+            # In the -remote use case, if the SourcePath and combined PlanDir/PlanPath are absolute paths in
+            # the model, we do not try to upload them and instead assume that they are available on the remote
+            # machine already.
+            #
+            deploy_should_upload = self.__remote_deploy_should_upload_app(app, app_name, is_library_module)
+            if deploy_should_upload:
+                deploy_options['upload'] = 'true'
+            else:
+                deploy_options['remote'] = 'true'
+        elif self.model_context.is_ssh():
             deploy_options['remote'] = 'true'
 
         if len(deploy_options) == 0:
@@ -1012,6 +1019,47 @@ def __get_deploy_options(self, model_apps, app_name, is_library_module):
                             result=[deploy_options, sub_module_targets])
         return deploy_options, sub_module_targets
 
+    def __remote_deploy_should_upload_app(self, deployment_dict, deployment_name, is_library_module):
+        """
+        In the -remote use case, should the deployment be uploaded>
+        :param deployment_dict: the model dictionary
+        :param deployment_name: the model name
+        :param is_library_module: whether the deployment is a shared library
+        :return: True, if the deploy options should set upload to true; False otherwise
+        :raises: DeployException: if the model deployment paths are inconsistent
+        """
+        _method_name = '__remote_deploy_should_upload_app'
+        self.logger.entering(deployment_dict, deployment_name, is_library_module,
+                             class_name=self._class_name, method_name=_method_name)
+
+        deploy_should_upload = True
+
+        model_source_path = dictionary_utils.get_element(deployment_dict, SOURCE_PATH)
+        if not string_utils.is_empty(model_source_path):
+            if self.path_helper.is_absolute_path(model_source_path):
+                deploy_should_upload = False
+
+        model_combined_plan_path = self._get_combined_model_plan_path(deployment_dict)
+        if not string_utils.is_empty(model_combined_plan_path):
+            if self.path_helper.is_absolute_path(model_combined_plan_path):
+                if string_utils.is_empty(model_source_path):
+                    deploy_should_upload = False
+                elif deploy_should_upload:
+                    # SourcePath was not empty and not an absolute path but the plan is an absolute path so
+                    # the model is inconsistent and the deploy operation cannot process it.
+                    #
+                    model_type = APPLICATION
+                    if is_library_module:
+                        model_type = LIBRARY
+                    ex = exception_helper.create_deploy_exception('WLSDPLY-09350', model_type, deployment_name,
+                                                                  model_source_path, model_combined_plan_path)
+                    self.logger.throwing(ex, class_name=self._class_name, method_name=_method_name)
+                    raise ex
+                # else deploy_should_upload already set to false so there is nothing to do
+
+        self.logger.exiting(class_name=self._class_name, method_name=_method_name, result=deploy_should_upload)
+        return deploy_should_upload
+
     def __set_sub_deployments_for_app_module(self, app, module_type, sub_module_targets):
         _method_name = '__set_sub_deployments_for_app_module'
         self.logger.entering(app, module_type, sub_module_targets, class_name=self._class_name, method_name=_method_name)

core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py

@@ -21,6 +21,9 @@
 from wlsdeploy.aliases.alias_constants import PASSWORD_TOKEN
 from wlsdeploy.aliases import model_constants
 from wlsdeploy.aliases.location_context import LocationContext
+from wlsdeploy.aliases.model_constants import PLAN_DIR
+from wlsdeploy.aliases.model_constants import PLAN_PATH
+from wlsdeploy.aliases.model_constants import SOURCE_PATH
 from wlsdeploy.aliases.wlst_modes import WlstModes
 from wlsdeploy.exception import exception_helper
 from wlsdeploy.logging.platform_logger import PlatformLogger
@@ -299,49 +302,48 @@ def _add_application_to_archive(self, application_name, application_dict):
         _logger.entering(application_name, class_name=_class_name, method_name=_method_name)
 
         archive_file = self._model_context.get_archive_file()
-        if model_constants.SOURCE_PATH in application_dict:
-            file_name = application_dict[model_constants.SOURCE_PATH]
-            if file_name:
-                file_name_path = file_name
-                if not self._model_context.is_remote():
-                    file_name_path = self._convert_path(file_name)
-                if self._is_file_to_exclude_from_archive(file_name_path):
-                    _logger.info('WLSDPLY-06393', application_name,
+        file_name = self._get_dictionary_attribute_with_path_tokens_replaced(application_dict, SOURCE_PATH)
+        if file_name is not None:
+            file_name_path = file_name
+            if not self._model_context.is_remote():
+                file_name_path = self._convert_path(file_name)
+            if self._is_file_to_exclude_from_archive(file_name_path):
+                _logger.info('WLSDPLY-06393', application_name,
+                             class_name=_class_name, method_name=_method_name)
+            else:
+                new_source_name = None
+                if self._model_context.is_remote():
+                    new_source_name = WLSDeployArchive.getApplicationArchivePath(file_name_path)
+                    self.add_to_remote_map(file_name_path, new_source_name,
+                                           WLSDeployArchive.ArchiveEntryType.APPLICATION.name())
+                elif not self._model_context.is_skip_archive():
+                    _logger.info('WLSDPLY-06394', application_name, file_name_path,
                                  class_name=_class_name, method_name=_method_name)
-                else:
-                    new_source_name = None
-                    if self._model_context.is_remote():
-                        new_source_name = WLSDeployArchive.getApplicationArchivePath(file_name_path)
-                        self.add_to_remote_map(file_name_path, new_source_name,
-                                               WLSDeployArchive.ArchiveEntryType.APPLICATION.name())
-                    elif not self._model_context.is_skip_archive():
-                        _logger.info('WLSDPLY-06394', application_name, file_name_path,
-                                     class_name=_class_name, method_name=_method_name)
-                        try:
-                            if self._model_context.is_ssh():
-                                file_name_path = \
-                                    self.download_deployment_from_remote_server(file_name_path,
-                                                                                self.download_temporary_dir,
-                                                                                "applications")
-
-                            new_source_name = archive_file.addApplication(file_name_path)
-                            module_type = dictionary_utils.get_dictionary_element(application_dict,
-                                                                                  model_constants.MODULE_TYPE)
-                            if module_type == 'jdbc':
-                                self._jdbc_password_fix(new_source_name)
-
-                        except IllegalArgumentException, iae:
-                            self._disconnect_target(application_name, application_dict, iae.getLocalizedMessage())
-                        except WLSDeployArchiveIOException, wioe:
-                            de = exception_helper.create_discover_exception('WLSDPLY-06397', application_name,
-                                                                        file_name_path, wioe.getLocalizedMessage())
-                            _logger.throwing(class_name=_class_name, method_name=_method_name, error=de)
-                            raise de
-                    if new_source_name is not None:
-                        _logger.finer('WLSDPLY-06398', application_name, new_source_name, class_name=_class_name,
-                                      method_name=_method_name)
-                        application_dict[model_constants.SOURCE_PATH] = new_source_name
-                    self.add_application_plan_to_archive(application_name, application_dict)
+                    try:
+                        if self._model_context.is_ssh():
+                            file_name_path = \
+                                self.download_deployment_from_remote_server(file_name_path,
+                                                                            self.download_temporary_dir,
+                                                                            "applications")
+
+                        new_source_name = archive_file.addApplication(file_name_path)
+                        module_type = dictionary_utils.get_dictionary_element(application_dict,
+                                                                              model_constants.MODULE_TYPE)
+                        if module_type == 'jdbc':
+                            self._jdbc_password_fix(new_source_name)
+
+                    except IllegalArgumentException, iae:
+                        self._disconnect_target(application_name, application_dict, iae.getLocalizedMessage())
+                    except WLSDeployArchiveIOException, wioe:
+                        de = exception_helper.create_discover_exception('WLSDPLY-06397', application_name,
+                                                                    file_name_path, wioe.getLocalizedMessage())
+                        _logger.throwing(class_name=_class_name, method_name=_method_name, error=de)
+                        raise de
+                if new_source_name is not None:
+                    _logger.finer('WLSDPLY-06398', application_name, new_source_name, class_name=_class_name,
+                                  method_name=_method_name)
+                    application_dict[model_constants.SOURCE_PATH] = new_source_name
+                self.add_application_plan_to_archive(application_name, application_dict)
 
         _logger.exiting(class_name=_class_name, method_name=_method_name)
 
@@ -357,9 +359,17 @@ def add_application_plan_to_archive(self, application_name, application_dict):
         _method_name = 'add_application_plan_to_archive'
         _logger.entering(application_name, class_name=_class_name, method_name=_method_name)
         archive_file = self._model_context.get_archive_file()
-        if model_constants.PLAN_PATH in application_dict:
-            app_source_name = application_dict[model_constants.SOURCE_PATH]
-            plan_path = application_dict[model_constants.PLAN_PATH]
+
+        model_plan_path = self._get_dictionary_attribute_with_path_tokens_replaced(application_dict, PLAN_PATH)
+        if model_plan_path is not None:
+            app_source_name = self._get_dictionary_attribute_with_path_tokens_replaced(application_dict, SOURCE_PATH)
+            model_plan_dir = self._get_dictionary_attribute_with_path_tokens_replaced(application_dict, PLAN_DIR)
+
+            if model_plan_dir is not None and self.path_helper.is_relative_path(model_plan_path):
+                plan_path = self.path_helper.join(model_plan_dir, model_plan_path)
+            else:
+                plan_path = model_plan_path
+
             if plan_path:
                 if not self._model_context.is_remote():
                     plan_path = self._convert_path(plan_path)
@@ -425,16 +435,9 @@ def _is_structured_app(self, application_name, application_dict):
         _method_name = '_is_structured_app'
         _logger.entering(application_dict, class_name=_class_name, method_name=_method_name)
 
-        source_path = None
-        plan_dir = None
-        plan_path = None
-
-        if 'SourcePath' in application_dict:
-            source_path = application_dict['SourcePath']
-        if 'PlanDir' in application_dict:
-            plan_dir = application_dict['PlanDir']
-        if 'PlanPath' in application_dict:
-            plan_path = application_dict['PlanPath']
+        source_path = self._get_dictionary_attribute_with_path_tokens_replaced(application_dict, SOURCE_PATH)
+        plan_dir = self._get_dictionary_attribute_with_path_tokens_replaced(application_dict, PLAN_DIR)
+        plan_path = self._get_dictionary_attribute_with_path_tokens_replaced(application_dict, PLAN_PATH)
 
         _logger.finer('WLSDPLY-06405', application_name, source_path, plan_dir, plan_path,
                       class_name=_class_name, method_name=_method_name)
@@ -681,6 +684,16 @@ def _get_app_install_root(self, app_dir, plan_dir):
         _logger.exiting(class_name=_class_name, method_name=_method_name, result=install_root)
         return install_root
 
+    def _get_dictionary_attribute_with_path_tokens_replaced(self, model_dict, attribute_name):
+        _method_name = '_get_dictionary_attribute_with_path_tokens_replaced'
+        _logger.entering(model_dict, attribute_name, class_name=_class_name, method_name=_method_name)
+
+        result = dictionary_utils.get_element(model_dict, attribute_name)
+        if result is not None:
+            result = self._model_context.replace_token_string(result)
+
+        _logger.exiting(class_name=_class_name, method_name=_method_name, result=result)
+        return result
 
 def _generate_new_plan_name(binary_path, plan_path):
     """

core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties

@@ -1436,6 +1436,7 @@ WLSDPLY-09348=Model application {0} is missing the SourcePath attribute but no e
   exact same name but one or more possible matches {1} were found.  Please update the model to either use the exact \
   name of application or add the SourcePath.
 WLSDPLY-09349=Cannot deploy structured application {0} with installation directory {1} with '-remote' option
+WLSDPLY-09350={0} {1} cannot be deployed using the -remote options since the SourcePath {2} and combined PlanDir and PlanPath {3} are not both absolute paths
 
 # wlsdeploy/tool/deploy/common_resources_deployer.py
 WLSDPLY-09400=ResourceGroup was specified in the test file but are not supported in WebLogic Server version {0}

documentation/4.0/content/release-notes/_index.md

@@ -21,7 +21,12 @@ pre = "<b> </b>"
                  Added online mode support.  Fixed CRDs to work consistently with alias folders and attributes.
 - #1550 - Refactored the archive file and moved content extraction into `$DOMAIN_HOME/config/wlsdeploy` for some types
           to take advantage of existing Pack/Unpack behavior and admin server to managed server replications capabilities.
-- #1584 - Added support for creating WebLogic authorization policies during domain creation (GitHub issue #1496). 
+- #1584 - Added support for creating WebLogic authorization policies during domain creation (GitHub issue #1496).
+- #1641, #1643 - Overhauled Application and Library provisioning.  As part of this overhaul, we have tried to define the
+          semantics for non-archive application deployments, particularly with online Update Domain and Deploy Apps tools.
+          For non-archive applications/libraries, online deployment of binaries outside of the archive will always
+          assume that the binaries are available to the Admin Server at the model-specified paths.  Neither the
+          `-remote` or SSH options will attempt to upload the non-archived binaries. 
 
 #### Other Changes
 - #1544 - Consolidated multiple internal WLST helper methods to get an MBean.
@@ -71,9 +76,12 @@ pre = "<b> </b>"
           WebLogic Server 14.1.1 and newer.
 - #1636 - Fixed an issue with Update Domain and Deploy Applications Tools when using the `-remote` option that was
           causing a TODO message to be generated when there was nothing for the user to do.
-- #1638 - Fixed an issue with Update Domain and Deploy Applications Tools where the application specified a `PlanDir`
-          and a `PlanPath` but the online deployment was ignoring the `PlanDir`, resulting in a file does not exist
-          error when attempting to deploy the application.
+- #1638 - Fixed an issue with the Update Domain and Deploy Applications Tools where the application specified a
+         `PlanDir` and a `PlanPath` but the online deployment was ignoring the `PlanDir`, resulting in a file does
+          not exist error when attempting to deploy the application.
+- #1642 - Fixed deployment issues with deploying applications not included in an archive file.
+- #1643 - Fixed an issue with Discover Domain where application/library path tokenization was preventing adding
+          deployments to the archive file.
 
 #### Known Issues
 - SSH support requires a reasonably recent version of Bouncy Castle.  WDT picks up Bouncy Castle from WLST so, for example,