[Enhancement][1875]zos_copy_implemented_async_support (#1953)

* Add first iteration

* Keep working

* Fix path

* Updated modules

* First iteration of solution

* Add fragment and change yml

* Add test case

* Change removal

* Remove removal

* Add debug

* Ensure properly clean up

* Fix finally

* Fix sanity and test

* Fix import

* Fix last import

* Update changelogs/fragments/1953-zos_copy_implemented_async_support.yml

Co-authored-by: Fernando Flores <[email protected]>

* fix space lint

* Fix positional

* Fix assertion

---------

Co-authored-by: Fernando Flores <[email protected]>

Author: AndreMarcel99

changelogs/fragments/1934-zos_script-implemented-async-support renamed to changelogs/fragments/1934-zos_script-implemented-async-support.yml

@@ -0,0 +1,3 @@
+minor_changes:
+  - zos_copy - Adds support for copying in asynchronous mode inside playbooks.
+    (https://github.com/ansible-collections/ibm_zos_core/pull/1953).

changelogs/fragments/1953-zos_copy_implemented_async_support.yml

@@ -28,10 +28,6 @@
 from ansible.utils.display import Display
 from ansible import cli
 
-from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.data_set import (
-    is_member
-)
-
 from ansible_collections.ibm.ibm_zos_core.plugins.module_utils import encode
 
 from ansible_collections.ibm.ibm_zos_core.plugins.module_utils import template
@@ -42,6 +38,8 @@
 class ActionModule(ActionBase):
     def run(self, tmp=None, task_vars=None):
         """ handler for file transfer operations """
+        self._supports_async = True
+
         if task_vars is None:
             task_vars = dict()
 
@@ -272,20 +270,14 @@ def run(self, tmp=None, task_vars=None):
                 encoding=encoding,
             )
         )
+
         copy_res = self._execute_module(
             module_name="ibm.ibm_zos_core.zos_copy",
             module_args=task_args,
             task_vars=task_vars,
+            wrap_async=self._task.async_val
         )
 
-        # Erasing all rendered Jinja2 templates from the controller.
-        if template_dir:
-            shutil.rmtree(template_dir, ignore_errors=True)
-        # Remove temporary directory from remote
-        if self.tmp_dir is not None:
-            path = os.path.normpath(f"{self.tmp_dir}/ansible-zos-copy")
-            self._connection.exec_command(f"rm -rf {path}*")
-
         if copy_res.get("note") and not force:
             result["note"] = copy_res.get("note")
             return result
@@ -304,10 +296,13 @@ def run(self, tmp=None, task_vars=None):
             )
             if backup or backup_name:
                 result["backup_name"] = copy_res.get("backup_name")
-            self._remote_cleanup(dest, copy_res.get("dest_exists"), task_vars)
             return result
 
-        return _update_result(is_binary, copy_res, self._task.args, original_src)
+        # Erasing all rendered Jinja2 templates from the controller.
+        if template_dir:
+            shutil.rmtree(template_dir, ignore_errors=True)
+
+        return copy_res
 
     def _copy_to_remote(self, src, is_dir=False, ignore_stderr=False):
         """Copy a file or directory to the remote z/OS system """
@@ -412,26 +407,6 @@ def _copy_to_remote(self, src, is_dir=False, ignore_stderr=False):
 
         return dict(temp_path=full_temp_path)
 
-    def _remote_cleanup(self, dest, dest_exists, task_vars):
-        """Remove all files or data sets pointed to by 'dest' on the remote
-        z/OS system. The idea behind this cleanup step is that if, for some
-        reason, the module fails after copying the data, we want to return the
-        remote system to its original state. Which means deleting any newly
-        created files or data sets.
-        """
-        if dest_exists is False:
-            if "/" in dest:
-                self._connection.exec_command("rm -rf {0}".format(dest))
-            else:
-                module_args = dict(name=dest, state="absent")
-                if is_member(dest):
-                    module_args["type"] = "member"
-                self._execute_module(
-                    module_name="ibm.ibm_zos_core.zos_data_set",
-                    module_args=module_args,
-                    task_vars=task_vars,
-                )
-
     def _exit_action(self, result, msg, failed=False):
         """Exit action plugin with a message"""
         result.update(
@@ -448,59 +423,6 @@ def _exit_action(self, result, msg, failed=False):
         return result
 
 
-def _update_result(is_binary, copy_res, original_args, original_src):
-    """ Helper function to update output result with the provided values """
-    ds_type = copy_res.get("ds_type")
-    src = copy_res.get("src")
-    note = copy_res.get("note")
-    backup_name = copy_res.get("backup_name")
-    dest_data_set_attrs = copy_res.get("dest_data_set_attrs")
-    updated_result = dict(
-        dest=copy_res.get("dest"),
-        is_binary=is_binary,
-        changed=copy_res.get("changed"),
-        invocation=dict(module_args=original_args),
-    )
-    if src:
-        updated_result["src"] = original_src
-    if note:
-        updated_result["note"] = note
-    if backup_name:
-        updated_result["backup_name"] = backup_name
-    if ds_type == "USS":
-        updated_result.update(
-            dict(
-                gid=copy_res.get("gid"),
-                uid=copy_res.get("uid"),
-                group=copy_res.get("group"),
-                owner=copy_res.get("owner"),
-                mode=copy_res.get("mode"),
-                state=copy_res.get("state"),
-                size=copy_res.get("size"),
-            )
-        )
-        checksum = copy_res.get("checksum")
-        if checksum:
-            updated_result["checksum"] = checksum
-    if dest_data_set_attrs is not None:
-        if len(dest_data_set_attrs) > 0:
-            dest_data_set_attrs.pop("name")
-            updated_result["dest_created"] = True
-            updated_result["destination_attributes"] = dest_data_set_attrs
-
-            # Setting attributes to lower case to conform to docs.
-            # Part of the change to lowercase choices in the collection involves having
-            # a consistent interface that also returns the same values in lowercase.
-            if "record_format" in updated_result["destination_attributes"]:
-                updated_result["destination_attributes"]["record_format"] = updated_result["destination_attributes"]["record_format"].lower()
-            if "space_type" in updated_result["destination_attributes"]:
-                updated_result["destination_attributes"]["space_type"] = updated_result["destination_attributes"]["space_type"].lower()
-            if "type" in updated_result["destination_attributes"]:
-                updated_result["destination_attributes"]["type"] = updated_result["destination_attributes"]["type"].lower()
-
-    return updated_result
-
-
 def _process_boolean(arg, default=False):
     try:
         return boolean(arg)

plugins/action/zos_copy.py

@@ -3199,6 +3199,75 @@ def normalize_line_endings(src, encoding=None):
     return src
 
 
+def remote_cleanup(module):
+    """Remove all files or data sets pointed to by 'dest' on the remote
+    z/OS system. The idea behind this cleanup step is that if, for some
+    reason, the module fails after copying the data, we want to return the
+    remote system to its original state. Which means deleting any newly
+    created files or data sets.
+    """
+    dest = module.params.get('dest')
+    if "/" in dest:
+        if os.path.isfile(dest):
+            os.remove(dest)
+        else:
+            shutil.rmtree(dest)
+    else:
+        dest = data_set.extract_dsname(dest)
+        data_set.DataSet.ensure_absent(name=dest)
+
+
+def update_result(res_args, original_args):
+    ds_type = res_args.get("ds_type")
+    src = res_args.get("src")
+    note = res_args.get("note")
+    backup_name = res_args.get("backup_name")
+    dest_data_set_attrs = res_args.get("dest_data_set_attrs")
+    updated_result = dict(
+        dest=res_args.get("dest"),
+        is_binary=original_args.get("is_binary"),
+        changed=res_args.get("changed"),
+        invocation=dict(module_args=original_args),
+    )
+    if src:
+        updated_result["src"] = original_args.get("src")
+    if note:
+        updated_result["note"] = note
+    if backup_name:
+        updated_result["backup_name"] = backup_name
+    if ds_type == "USS":
+        updated_result.update(
+            dict(
+                gid=res_args.get("gid"),
+                uid=res_args.get("uid"),
+                group=res_args.get("group"),
+                owner=res_args.get("owner"),
+                mode=res_args.get("mode"),
+                state=res_args.get("state"),
+                size=res_args.get("size"),
+            )
+        )
+        checksum = res_args.get("checksum")
+        if checksum:
+            updated_result["checksum"] = checksum
+    if dest_data_set_attrs is not None:
+        if len(dest_data_set_attrs) > 0:
+            dest_data_set_attrs.pop("name")
+            updated_result["dest_created"] = True
+            updated_result["destination_attributes"] = dest_data_set_attrs
+
+            # Setting attributes to lower case to conform to docs.
+            # Part of the change to lowercase choices in the collection involves having
+            # a consistent interface that also returns the same values in lowercase.
+            if "record_format" in updated_result["destination_attributes"]:
+                updated_result["destination_attributes"]["record_format"] = updated_result["destination_attributes"]["record_format"].lower()
+            if "space_type" in updated_result["destination_attributes"]:
+                updated_result["destination_attributes"]["space_type"] = updated_result["destination_attributes"]["space_type"].lower()
+            if "type" in updated_result["destination_attributes"]:
+                updated_result["destination_attributes"]["type"] = updated_result["destination_attributes"]["type"].lower()
+    return updated_result
+
+
 def run_module(module, arg_def):
     """Initialize module
 
@@ -4047,9 +4116,24 @@ def main():
     res_args = conv_path = None
     try:
         res_args, conv_path = run_module(module, arg_def)
+
+        # Verification of default tmpdir use by the collection to remove
+        path = str(module.tmpdir)
+        position = path[:-1].rfind('/')
+        tmp_dir = path[:position]
+
+        default_path = os.path.normpath(f"{tmp_dir}/ansible-zos-copy")
+
+        if os.path.exists(path):
+            shutil.rmtree(path)
+        elif os.path.exists(default_path):
+            shutil.rmtree(default_path)
+
+        res_args = update_result(res_args=res_args, original_args=module.params)
         module.exit_json(**res_args)
     except CopyOperationError as err:
         cleanup([])
+        remote_cleanup(module=module)
         module.fail_json(**(err.json_args))
     finally:
         cleanup([conv_path])

plugins/modules/zos_copy.py

@@ -18,8 +18,10 @@
 import shutil
 import re
 import time
+import yaml
 import tempfile
 import subprocess
+from shellescape import quote
 
 from ibm_zos_core.tests.helpers.volumes import Volume_Handler
 from ibm_zos_core.tests.helpers.dataset import get_tmp_ds_name
@@ -222,6 +224,54 @@
 //STDERR DD SYSOUT=*
 //"""
 
+PLAYBOOK_ASYNC_TEST = """- hosts: zvm
+  collections:
+    - ibm.ibm_zos_core
+  gather_facts: False
+  environment:
+    _BPXK_AUTOCVT: "ON"
+    ZOAU_HOME: "{0}"
+    PYTHONPATH: "{0}/lib/{2}"
+    LIBPATH: "{0}/lib:{1}/lib:/lib:/usr/lib:."
+    PATH: "{0}/bin:/bin:/usr/lpp/rsusr/ported/bin:/var/bin:/usr/lpp/rsusr/ported/bin:/usr/lpp/java/java180/J8.0_64/bin:{1}/bin:"
+    _CEE_RUNOPTS: "FILETAG(AUTOCVT,AUTOTAG) POSIX(ON)"
+    _TAG_REDIR_ERR: "txt"
+    _TAG_REDIR_IN: "txt"
+    _TAG_REDIR_OUT: "txt"
+    LANG: "C"
+
+  tasks:
+    - name: Create a copy
+      zos_copy:
+            src: /etc/profile
+            remote_src: True
+            force: True
+            dest: {3}
+      async: 50
+      poll: 0
+      register: copy_output
+
+    - name: Query async task.
+      async_status:
+        jid: "{{{{ copy_output.ansible_job_id }}}}"
+      register: job_result
+      until: job_result.finished
+      retries: 10
+      delay: 30
+
+    - name: Echo copy_output.
+      debug:
+        msg: "{{ job_result }}"
+"""
+
+INVENTORY_ASYNC_TEST = """all:
+  hosts:
+    zvm:
+      ansible_host: {0}
+      ansible_ssh_private_key_file: {1}
+      ansible_user: {2}
+      ansible_python_interpreter: {3}"""
+
 def populate_dir(dir_path):
     for i in range(5):
         with open(dir_path + "/" + "file" + str(i + 1), "w") as infile:
@@ -637,6 +687,7 @@ def test_copy_dir_to_existing_uss_dir_not_forced(ansible_zos_module):
         )
 
         for result in copy_result.contacted.values():
+            print(result)
             assert result.get("msg") is not None
             assert result.get("changed") is False
             assert "Error" in result.get("msg")
@@ -5514,3 +5565,61 @@ def test_copy_to_dataset_with_special_symbols(ansible_zos_module):
     finally:
         hosts.all.zos_data_set(name=src_data_set, state="absent")
         hosts.all.zos_data_set(name=dest_data_set, state="absent")
+
+
+def test_job_script_async(ansible_zos_module, get_config):
+    try:
+        ds_name = get_tmp_ds_name()
+        path = get_config
+        with open(path, 'r') as file:
+            enviroment = yaml.safe_load(file)
+
+        ssh_key = enviroment["ssh_key"]
+        hosts = enviroment["host"].upper()
+        user = enviroment["user"].upper()
+        python_path = enviroment["python_path"]
+        cut_python_path = python_path[:python_path.find('/bin')].strip()
+        zoau = enviroment["environment"]["ZOAU_ROOT"]
+        python_version = cut_python_path.split('/')[2]
+
+        playbook = tempfile.NamedTemporaryFile(delete=True)
+        inventory = tempfile.NamedTemporaryFile(delete=True)
+
+        os.system("echo {0} > {1}".format(
+            quote(PLAYBOOK_ASYNC_TEST.format(
+                zoau,
+                cut_python_path,
+                python_version,
+                ds_name
+            )),
+            playbook.name
+        ))
+
+        os.system("echo {0} > {1}".format(
+            quote(INVENTORY_ASYNC_TEST.format(
+                hosts,
+                ssh_key,
+                user,
+                python_path
+            )),
+            inventory.name
+        ))
+
+        command = "ansible-playbook -i {0} {1}".format(
+            inventory.name,
+            playbook.name
+        )
+
+        result = subprocess.run(
+            command,
+            capture_output=True,
+            shell=True,
+            timeout=120,
+            encoding='utf-8'
+        )
+        assert result.returncode == 0
+        assert "ok=3" in result.stdout
+        assert "changed=2" in result.stdout
+        assert result.stderr == ""
+    finally:
+        ansible_zos_module.all.zos_data_set(name=ds_name, state="absent")