Use datasets.write to write JCL instead of cp (#170)

Use datasets.write to write JCL instead of cp

---------

Signed-off-by: Stewart Francis <[email protected]>

Author: stewartfrancis

changelogs/fragments/170-fix-jcl-alloc.yaml

@@ -0,0 +1,2 @@
+bugfixes:
+  - region_jcl - Use ZOAU API to write JCL to a data set, so that sequential data sets are allocated properly.  The resulting data set can now be submitted from ISPF. (https://github.com/ansible-collections/ibm_zos_cics/pull/170).

plugins/module_utils/_data_set_utils.py

@@ -9,7 +9,6 @@
 
 __metaclass__ = type
 import re
-import tempfile
 from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.ansible_module import AnsibleModuleHelper
 from ansible_collections.ibm.ibm_zos_cics.plugins.module_utils._response import _execution, MVSExecutionException
 from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.zos_mvs_raw import MVSCmd
@@ -290,23 +289,3 @@ def _read_data_set_content(data_set_name):
             "RC {0} when reading content from data set {1}".format(
                 rc, data_set_name), executions)
     return executions, stdout
-
-
-def _write_jcl_to_data_set(jcl, data_set_name):
-    """Writes generated JCL content to the specified data set
-    """
-    executions = []
-
-    temp = tempfile.NamedTemporaryFile(delete=True)
-    with open(temp.name, "w") as f:
-        f.write(jcl)
-    rc, stdout, stderr = _execute_command("cp -O u {0} \"//'{1}'\"".format(temp.name, data_set_name))
-    executions.append(
-        _execution(
-            name="Copy JCL contents to data set",
-            rc=rc,
-            stdout=stdout,
-            stderr=stderr))
-    if rc != 0:
-        raise MVSExecutionException("Failed to copy JCL content to data set", executions)
-    return executions

plugins/module_utils/_response.py

@@ -19,6 +19,6 @@ def _execution(name, rc, stdout, stderr):  # type: (str, int, str, str) -> dict
 
 
 class MVSExecutionException(Exception):
-    def __init__(self, message, executions):   # type: (str, list[_execution]) -> None
+    def __init__(self, message, executions):   # type: (str, list[dict]) -> None
         self.message = message
         self.executions = executions

plugins/modules/region_jcl.py

@@ -185,6 +185,8 @@
 
 import string
 import math
+import traceback
+
 from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.data_set import is_member
 from ansible_collections.ibm.ibm_zos_cics.plugins.module_utils._data_set import (
     MEGABYTES,
@@ -196,14 +198,32 @@
     WARM,
     DataSet
 )
-from ansible_collections.ibm.ibm_zos_cics.plugins.module_utils._data_set_utils import _read_data_set_content, _write_jcl_to_data_set
+from ansible_collections.ibm.ibm_zos_cics.plugins.module_utils._data_set_utils import _read_data_set_content
 from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.dd_statement import DatasetDefinition
 from ansible_collections.ibm.ibm_zos_cics.plugins.module_utils._jcl_helper import (
     JCLHelper, DLM, DD_INSTREAM, CONTENT, END_INSTREAM, JOB_CARD, EXECS, JOB_NAME, DDS, NAME
 )
-from ansible_collections.ibm.ibm_zos_cics.plugins.module_utils._response import MVSExecutionException
+from ansible_collections.ibm.ibm_zos_cics.plugins.module_utils._response import (
+    MVSExecutionException,
+    _execution
+)
 from ansible_collections.ibm.ibm_zos_cics.plugins.module_utils._data_set_utils import _run_listds
 
+from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.import_handler import (
+    ZOAUImportError
+)
+
+try:
+    from zoautil_py import datasets, exceptions
+except Exception:
+    # Use ibm_zos_core's approach to handling zoautil_py imports so sanity tests pass
+    datasets = ZOAUImportError(traceback.format_exc())
+    exceptions = ZOAUImportError(traceback.format_exc())
+
+from ansible_collections.ibm.ibm_zos_cics.plugins.module_utils._response import (
+    MVSExecutionException,
+)
+
 
 DFHSTART = "dfhstart"
 SPACE_PRIMARY_DEFAULT = 5
@@ -354,7 +374,7 @@ def create_data_set(self):  # type: () -> None
                 primary_unit=self.primary_unit,
                 secondary_unit=self.secondary_unit,
                 volumes=self.volumes,
-                block_size=4096,
+                block_size=32720,
                 record_length=80,
                 record_format="FB",
                 disposition="NEW",
@@ -373,7 +393,7 @@ def generate_jcl(self):
 
     def write_jcl(self):
         try:
-            jcl_writer_execution = _write_jcl_to_data_set(self.jcl, self.name)
+            jcl_writer_execution = self._write_jcl_to_data_set(self.jcl, self.name)
             self.executions.extend(jcl_writer_execution)
             self.changed = True
         except MVSExecutionException as e:
@@ -671,6 +691,35 @@ def _find_sit_parm_key(input_string):
         else:
             return None
 
+    @staticmethod
+    def _write_jcl_to_data_set(jcl, data_set_name):
+        """Writes generated JCL content to the specified data set
+        """
+        executions = []
+
+        try:
+            rc = datasets.write(data_set_name, jcl)
+            # If rc != 0, ZOAU raises an exception
+            executions.append(
+                _execution(
+                    name="Copy JCL contents to data set",
+                    rc=rc,
+                    stdout="",
+                    stderr=""
+                )
+            )
+        except exceptions.DatasetWriteException as e:
+            raise MVSExecutionException("Failed to copy JCL content to data set", [
+                _execution(
+                    name="Copy JCL contents to data set",
+                    rc=e.response.rc,
+                    stdout=e.response.stdout_response,
+                    stderr=e.response.stderr_response
+                )
+            ])
+
+        return executions
+
     @staticmethod
     def init_argument_spec():  # type: () -> dict
         return {

tests/integration/targets/cics_region_jcl/playbooks/region_jcl_seq.yml

@@ -0,0 +1,74 @@
+# (c) Copyright IBM Corp. 2024
+# Apache License, Version 2.0 (see https://opensource.org/licenses/Apache-2.0)
+---
+- name: Test lifecycle of sequential data set for region_jcl module
+
+  hosts: "all"
+  gather_facts: false
+  environment: "{{ environment_vars }}"
+  vars:
+    base_data_set_name: SEQJCL
+    data_set_path: "{{ region_data_set_path }}.{{ base_data_set_name }}"
+
+  module_defaults:
+    ibm.ibm_zos_cics.region_jcl:
+      region_data_sets:
+        template: "{{ region_data_set_path }}.<< data_set_name >>"
+        dfhstart:
+          dsn: "{{ data_set_path }}"
+      cics_data_sets:
+        template: "{{ cics_install_path }}.<< lib_name >>"
+        sdfhlic: "{{ cics_install_path }}.LIC.SDFHLIC"
+      le_data_sets:
+        template: "{{ le_path }}.<< lib_name >>"
+      applid: "{{ start_region_applid }}"
+
+  tasks:
+    # #############################################################################
+    # ############################## Initial Cleanup ##############################
+    # #############################################################################
+
+    - name: Delete base data set {{ base_data_set_path }}
+      ibm.ibm_zos_core.zos_data_set:
+        name: "{{ base_data_set_path }}"
+        state: absent
+      register: result
+      retries: 3
+      until: result is not failed
+
+    # #############################################################################
+    # ############################## Module Testing ###############################
+    # #############################################################################
+
+    - name: Wrap test in block so cleanup always runs
+      block:
+        - name: Run region_jcl module with initial state for exisitng PDS
+          ibm.ibm_zos_cics.region_jcl:
+            state: initial
+          register: result
+
+        - name: Debug
+          ansible.builtin.debug:
+            msg: "{{ result }}"
+
+        - name: Assert {{ data_set_path }} created
+          ansible.builtin.assert:
+            that:
+              - result.failed == false
+              - result.changed == true
+              - result.start_state.exists == false
+              - result.end_state.exists == true
+              - result.msg == ""
+
+        # #############################################################################
+        # ################################## Cleanup ##################################
+        # #############################################################################
+
+      always:
+        - name: Delete base data set {{ base_data_set_path }}
+          ibm.ibm_zos_core.zos_data_set:
+            name: "{{ base_data_set_path }}"
+            state: absent
+          register: result
+          retries: 3
+          until: result is not failed

tests/unit/module_utils/test_dataset_utils.py

@@ -657,46 +657,3 @@ def test__read_data_set_content_bad_rc():
         assert e.executions == expected_executions
     else:
         assert False
-
-
-def test__write_jcl_to_data_set():
-    data_set_name = "TEST.DATA.SET"
-    jcl = ""
-
-    rc = 0
-    stdout = ""
-    stderr = ""
-    data_set_utils._execute_command = MagicMock(return_value=(rc, stdout, stderr))
-
-    expected_executions = [{
-        "name": "Copy JCL contents to data set",
-        "rc": 0,
-        "stdout": "",
-        "stderr": ""
-    }]
-
-    executions = data_set_utils._write_jcl_to_data_set(jcl, data_set_name)
-
-    assert expected_executions == executions
-
-
-def test__write_jcl_to_data_set_fail():
-    data_set_name = "TEST.DATA.SET"
-    jcl = ""
-    rc = 99
-    stdout = "cp failed"
-    stderr = "stderr"
-    data_set_utils._execute_command = MagicMock(return_value=(rc, stdout, stderr))
-
-    expected_executions = [{
-        "name": "Copy JCL contents to data set",
-        "rc": 99,
-        "stdout": "cp failed",
-        "stderr": "stderr"
-    }]
-
-    with pytest.raises(MVSExecutionException) as e:
-        data_set_utils._write_jcl_to_data_set(jcl, data_set_name)
-
-    assert e.value.message == "Failed to copy JCL content to data set"
-    assert e.value.executions == expected_executions

tests/unit/modules/test_region_jcl.py

@@ -1072,7 +1072,14 @@ def test_initial_state():
         )
     )
 
-    _data_set_utils._execute_command = MagicMock(return_value=(0, "", ""))
+    region_jcl_module._write_jcl_to_data_set = MagicMock(return_value=[
+        _execution(
+            name="Copy JCL contents to data set",
+            rc=0,
+            stdout="",
+            stderr=""
+        )
+    ])
 
     region_jcl_module.main()
     expected_result = dict(
@@ -1138,7 +1145,14 @@ def test_initial_state_pre_existing():
         )
     )
 
-    _data_set_utils._execute_command = MagicMock(return_value=(0, "", ""))
+    region_jcl_module._write_jcl_to_data_set = MagicMock(return_value=[
+        _execution(
+            name="Copy JCL contents to data set",
+            rc=0,
+            stdout="",
+            stderr=""
+        )
+    ])
 
     region_jcl_module.main()
     expected_result = dict(
@@ -1430,7 +1444,14 @@ def test_initial_state_member():
         ]
     )
 
-    _data_set_utils._execute_command = MagicMock(return_value=(0, "", ""))
+    region_jcl_module._write_jcl_to_data_set = MagicMock(return_value=[
+        _execution(
+            name="Copy JCL contents to data set",
+            rc=0,
+            stdout="",
+            stderr=""
+        )
+    ])
 
     region_jcl_module.main()
     expected_result = dict(
@@ -1550,7 +1571,14 @@ def test_initial_state_pre_existing_member():
         return_value=MVSCmdResponse(0, IDCAMS_delete(MEMBER_DS_NAME), "")
     )
 
-    _data_set_utils._execute_command = MagicMock(return_value=(0, "", ""))
+    region_jcl_module._write_jcl_to_data_set = MagicMock(return_value=[
+        _execution(
+            name="Copy JCL contents to data set",
+            rc=0,
+            stdout="",
+            stderr=""
+        )
+    ])
 
     region_jcl_module.main()
     expected_result = dict(