[Enhancement] [zos_job_submit] Async support for zos_job_submit (#1786)

* Add async support for zos_copy

* Add use of remote_tmp in action plugin

* Remove use of ansible's tempfile

* Fix temp file path

* Add async support to zos_job_submit

* Update removal of temp files

* Update authors list

* Add test for async support

* Remove async support from zos_copy

* Remove unnecessary blank line

* Add changelog fragment

* Fix playbook used in async test

* Fix inventory used in async test

* Remove commented code

* Update test

Author: rexemin

changelogs/fragments/1786-zos_job_submit-async-support.yml

@@ -0,0 +1,4 @@
+minor_changes:
+  - zos_job_submit - Added support to run zos_job_submit tasks in async mode inside
+    playbooks.
+    (https://github.com/ansible-collections/ibm_zos_core/pull/1786).

plugins/action/zos_job_submit.py

@@ -23,13 +23,17 @@
 
 from ansible_collections.ibm.ibm_zos_core.plugins.module_utils import template
 
+from datetime import datetime
+from os import path
 
 display = Display()
 
 
 class ActionModule(ActionBase):
     def run(self, tmp=None, task_vars=None):
         """ handler for file transfer operations """
+        self._supports_async = True
+
         if task_vars is None:
             task_vars = {}
 
@@ -54,22 +58,9 @@ def run(self, tmp=None, task_vars=None):
 
             source = self._task.args.get("src", None)
 
-            # Get a temporary file on the managed node
-            tempfile = self._execute_module(
-                module_name="tempfile", module_args={}, task_vars=task_vars,
-            )
-            dest_path = tempfile.get("path")
-            # Calling execute_module from this step with tempfile leaves behind a tmpdir.
-            # This is called to ensure the proper removal.
-            tmpdir = self._connection._shell.tmpdir
-            if tmpdir:
-                self._remove_tmp_path(tmpdir)
-
             result["failed"] = True
             if source is None:
                 result["msg"] = "Source is required."
-            elif dest_path is None:
-                result["msg"] = "Failed copying to remote, destination file was not created. {0}".format(tempfile.get("msg"))
             elif source is not None and os.path.isdir(to_bytes(source, errors="surrogate_or_strict")):
                 result["msg"] = "Source must be a file."
             else:
@@ -85,26 +76,27 @@ def run(self, tmp=None, task_vars=None):
                 result["msg"] = to_text(e)
                 return result
 
-            if tmp is None or "-tmp-" not in tmp:
-                tmp = self._make_tmp_path()
+            tmp_dir = self._connection._shell._options.get("remote_tmp")
+            rc, stdout, stderr = self._connection.exec_command("cd {0} && pwd".format(tmp_dir))
+            if rc > 0:
+                msg = f"Failed to resolve remote temporary directory {tmp_dir}. Ensure that the directory exists and user has proper access."
+                return self._exit_action({}, msg, failed=True)
+
+            tmp_dir = stdout.decode("utf-8").replace("\r", "").replace("\n", "")
+            temp_file_dir = f'zos_job_submit_{datetime.now().strftime("%Y%m%d%S%f")}'
+            dest_path = path.join(tmp_dir, temp_file_dir, path.basename(source))
+            # Creating the name for the temp file needed.
+            self._connection.exec_command("mkdir -p {0}".format(path.dirname(dest_path)))
 
             source_full = None
             try:
                 source_full = self._loader.get_real_file(source)
-                # source_rel = os.path.basename(source)
             except AnsibleFileNotFound as e:
                 result["failed"] = True
                 result["msg"] = "Source {0} not found. {1}".format(source_full, e)
                 self._remove_tmp_path(tmp)
                 return result
 
-            # if self._connection._shell.path_has_trailing_slash(dest):
-            #     dest_file = self._connection._shell.join_path(dest, source_rel)
-            # else:
-            self._connection._shell.join_path(dest_path)
-
-            tmp_src = self._connection._shell.join_path(tmp, "source")
-
             rendered_file = None
             if use_template:
                 template_parameters = module_args.get("template_parameters", dict())
@@ -129,36 +121,35 @@ def run(self, tmp=None, task_vars=None):
 
                 source_full = rendered_file
 
-            remote_path = None
-            remote_path = self._transfer_file(source_full, tmp_src)
-
-            if remote_path:
-                self._fixup_perms2((tmp, remote_path))
-
             result = {}
             copy_module_args = {}
             module_args = self._task.args.copy()
 
             copy_module_args.update(
                 dict(
-                    src=tmp_src,
+                    src=source_full,
                     dest=dest_path,
                     mode="0600",
                     force=True,
                     encoding=module_args.get('encoding'),
-                    remote_src=True,
+                    remote_src=False,
                 )
             )
             copy_task = self._task.copy()
             copy_task.args = copy_module_args
+            # Making the zos_copy task run synchronously every time.
+            copy_task.async_val = 0
+
             copy_action = self._shared_loader_obj.action_loader.get(
                 'ibm.ibm_zos_core.zos_copy',
                 task=copy_task,
                 connection=self._connection,
                 play_context=self._play_context,
                 loader=self._loader,
                 templar=self._templar,
-                shared_loader_obj=self._shared_loader_obj)
+                shared_loader_obj=self._shared_loader_obj
+            )
+
             result.update(copy_action.run(task_vars=task_vars))
             if result.get("msg") is None:
                 module_args["src"] = dest_path
@@ -167,6 +158,7 @@ def run(self, tmp=None, task_vars=None):
                         module_name="ibm.ibm_zos_core.zos_job_submit",
                         module_args=module_args,
                         task_vars=task_vars,
+                        wrap_async=self._task.async_val
                     )
                 )
             else:
@@ -178,6 +170,7 @@ def run(self, tmp=None, task_vars=None):
                     module_name="ibm.ibm_zos_core.zos_job_submit",
                     module_args=module_args,
                     task_vars=task_vars,
+                    wrap_async=self._task.async_val
                 )
             )
 

plugins/modules/zos_job_submit.py

@@ -23,6 +23,7 @@
     - "Xiao Yuan Ma (@bjmaxy)"
     - "Rich Parker (@richp405)"
     - "Demetrios Dimatos (@ddimatos)"
+    - "Ivan Moreno (@rexemin)"
 short_description: Submit JCL
 description:
     - Submit JCL in a data set, USS file, or file on the controller.
@@ -657,7 +658,8 @@
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils._text import to_text
 from timeit import default_timer as timer
-from os import remove
+from os import path
+import shutil
 import traceback
 from time import sleep
 import re
@@ -1202,7 +1204,7 @@ def run_module():
 
         finally:
             if temp_file is not None:
-                remove(temp_file)
+                shutil.rmtree(path.dirname(temp_file))
 
     # If max_rc is set, we don't want to default to changed=True, rely on 'is_changed'
     result["changed"] = True if is_changed else False

tests/functional/modules/test_zos_job_submit_func.py

@@ -18,9 +18,11 @@
 import tempfile
 import re
 import os
+import yaml
 from shellescape import quote
 import pytest
 from datetime import datetime
+import subprocess
 
 from ibm_zos_core.tests.helpers.volumes import Volume_Handler
 from ibm_zos_core.tests.helpers.dataset import get_tmp_ds_name
@@ -381,6 +383,48 @@
 //
 """
 
+PLAYBOOK_ASYNC_TEST = """- hosts: zvm
+  collections:
+    - ibm.ibm_zos_core
+  gather_facts: False
+  environment:
+    _BPXK_AUTOCVT: "ON"
+    ZOAU_HOME: "{0}"
+    PYTHONPATH: "{0}/lib/{2}"
+    LIBPATH: "{0}/lib:{1}/lib:/lib:/usr/lib:."
+    PATH: "{0}/bin:/bin:/usr/lpp/rsusr/ported/bin:/var/bin:/usr/lpp/rsusr/ported/bin:/usr/lpp/java/java180/J8.0_64/bin:{1}/bin:"
+    _CEE_RUNOPTS: "FILETAG(AUTOCVT,AUTOTAG) POSIX(ON)"
+    _TAG_REDIR_ERR: "txt"
+    _TAG_REDIR_IN: "txt"
+    _TAG_REDIR_OUT: "txt"
+    LANG: "C"
+
+  tasks:
+    - name: Submit async job.
+      ibm.ibm_zos_core.zos_job_submit:
+        src: {3}
+        location: local
+      async: 45
+      poll: 0
+      register: job_task
+
+    - name: Query async task.
+      async_status:
+        jid: "{{{{ job_task.ansible_job_id }}}}"
+      register: job_result
+      until: job_result.finished
+      retries: 20
+      delay: 5
+"""
+
+INVENTORY_ASYNC_TEST = """all:
+  hosts:
+    zvm:
+      ansible_host: {0}
+      ansible_ssh_private_key_file: {1}
+      ansible_user: {2}
+      ansible_python_interpreter: {3}"""
+
 
 @pytest.mark.parametrize(
     "location", [
@@ -1113,3 +1157,65 @@ def test_zoau_bugfix_invalid_utf8_chars(ansible_zos_module):
             assert result.get("changed") is True
     finally:
         hosts.all.file(path=temp_path, state="absent")
+
+
+def test_job_submit_async(get_config):
+    # Creating temp JCL file used by the playbook.
+    tmp_file = tempfile.NamedTemporaryFile(delete=True)
+    with open(tmp_file.name, "w",encoding="utf-8") as f:
+        f.write(JCL_FILE_CONTENTS)
+
+    # Getting all the info required to run the playbook.
+    path = get_config
+    with open(path, 'r') as file:
+        enviroment = yaml.safe_load(file)
+
+    ssh_key = enviroment["ssh_key"]
+    hosts = enviroment["host"].upper()
+    user = enviroment["user"].upper()
+    python_path = enviroment["python_path"]
+    cut_python_path = python_path[:python_path.find('/bin')].strip()
+    zoau = enviroment["environment"]["ZOAU_ROOT"]
+    python_version = cut_python_path.split('/')[2]
+
+    playbook = tempfile.NamedTemporaryFile(delete=True)
+    inventory = tempfile.NamedTemporaryFile(delete=True)
+
+    os.system("echo {0} > {1}".format(
+        quote(PLAYBOOK_ASYNC_TEST.format(
+            zoau,
+            cut_python_path,
+            python_version,
+            tmp_file.name
+        )), 
+        playbook.name
+    ))
+
+    os.system("echo {0} > {1}".format(
+        quote(INVENTORY_ASYNC_TEST.format(
+            hosts,
+            ssh_key,
+            user,
+            python_path
+        )), 
+        inventory.name
+    ))
+
+    command = "ansible-playbook -i {0} {1}".format(
+        inventory.name,
+        playbook.name
+    )
+
+    result = subprocess.run(
+        command,
+        capture_output=True,
+        shell=True,
+        timeout=120,
+        encoding='utf-8'
+    )
+
+    assert result.returncode == 0
+    assert "ok=2" in result.stdout
+    assert "changed=2" in result.stdout
+    assert result.stderr == ""
+