Merge branch 'dev' into enabler/update-test-suits-pylint/zos-copy-func

Author: AndreMarcel99

changelogs/fragments/1519-zos_fetch-gdg-support.yml

@@ -0,0 +1,4 @@
+minor_changes:
+  - zos_fetch - add support for fetching generation data groups and
+    generation data sets.
+    (https://github.com/ansible-collections/ibm_zos_core/pull/1519)

changelogs/fragments/1525-mvs_raw_support_gdg_gds_special_character.yml

@@ -0,0 +1,4 @@
+minor_changes:
+  - zos_mvs_raw - Added support for GDG and GDS relative name notation to use a data set.
+    Added support for data set names with special characters like $, /#, /- and @.
+    (https://github.com/ansible-collections/ibm_zos_core/pull/1525).

changelogs/fragments/1527-zos_backup-gdg.yml

@@ -0,0 +1,5 @@
+minor_changes:
+  - zos_data_set - Added support for GDS relative name notation to include or exclude data sets when
+    operation is backup. Added support for data set names with special characters
+    like $, /#, and @.
+    (https://github.com/ansible-collections/ibm_zos_core/pull/1527).

docs/source/modules/zos_fetch.rst

@@ -16,11 +16,13 @@ zos_fetch -- Fetch data from z/OS
 
 Synopsis
 --------
-- This module fetches a UNIX System Services (USS) file, PS (sequential data set), PDS, PDSE, member of a PDS or PDSE, or KSDS (VSAM data set) from a remote z/OS system.
+- This module fetches a UNIX System Services (USS) file, PS (sequential data set), PDS, PDSE, member of a PDS or PDSE, generation data set (GDS), generation data group (GDG), or KSDS (VSAM data set) from a remote z/OS system.
 - When fetching a sequential data set, the destination file name will be the same as the data set name.
 - When fetching a PDS or PDSE, the destination will be a directory with the same name as the PDS or PDSE.
 - When fetching a PDS/PDSE member, destination will be a file.
 - Files that already exist at \ :literal:`dest`\  will be overwritten if they are different than \ :literal:`src`\ .
+- When fetching a GDS, the relative name will be resolved to its absolute one.
+- When fetching a generation data group, the destination will be a directory with the same name as the GDG.
 
 
 
@@ -31,7 +33,7 @@ Parameters
 
 
 src
-  Name of a UNIX System Services (USS) file, PS (sequential data set), PDS, PDSE, member of a PDS, PDSE or KSDS (VSAM data set).
+  Name of a UNIX System Services (USS) file, PS (sequential data set), PDS, PDSE, member of a PDS, PDSE, GDS, GDG or KSDS (VSAM data set).
 
   USS file paths should be absolute paths.
 
@@ -187,6 +189,24 @@ Examples
          to: ISO8859-1
        flat: true
 
+   - name: Fetch the current generation data set from a GDG
+     zos_fetch:
+       src: USERHLQ.DATA.SET(0)
+       dest: /tmp/
+       flat: true
+
+   - name: Fetch a previous generation data set from a GDG
+     zos_fetch:
+       src: USERHLQ.DATA.SET(-3)
+       dest: /tmp/
+       flat: true
+
+   - name: Fetch a generation data group
+     zos_fetch:
+       src: USERHLQ.TEST.GDG
+       dest: /tmp/
+       flat: true
+
 
 
 

plugins/action/zos_fetch.py

@@ -26,9 +26,15 @@
 from ansible.utils.display import Display
 from ansible import cli
 
-from ansible_collections.ibm.ibm_zos_core.plugins.module_utils import encode, validation
+from ansible_collections.ibm.ibm_zos_core.plugins.module_utils import encode, validation, data_set
 
-SUPPORTED_DS_TYPES = frozenset({"PS", "PO", "VSAM", "USS"})
+SUPPORTED_DS_TYPES = frozenset({
+    "PS", "SEQ", "BASIC",
+    "PO", "PE", "PDS", "PDSE",
+    "VSAM", "KSDS",
+    "GDG",
+    "USS"
+})
 
 display = Display()
 
@@ -37,10 +43,15 @@ def _update_result(result, src, dest, ds_type="USS", is_binary=False):
     """ Helper function to update output result with the provided values """
     data_set_types = {
         "PS": "Sequential",
+        "SEQ": "Sequential",
+        "BASIC": "Sequential",
         "PO": "Partitioned",
-        "PDSE": "Partitioned Extended",
+        "PDS": "Partitioned",
         "PE": "Partitioned Extended",
+        "PDSE": "Partitioned Extended",
         "VSAM": "VSAM",
+        "KSDS": "VSAM",
+        "GDG": "Generation Data Group",
         "USS": "USS",
     }
     updated_result = dict((k, v) for k, v in result.items())
@@ -101,6 +112,7 @@ class ActionModule(ActionBase):
     def run(self, tmp=None, task_vars=None):
         result = super(ActionModule, self).run(tmp, task_vars)
         del tmp
+
         # ********************************************************** #
         #                 Parameter initializations                  #
         # ********************************************************** #
@@ -139,12 +151,60 @@ def run(self, tmp=None, task_vars=None):
             return result
 
         ds_type = None
-        fetch_member = "(" in src and src.endswith(")")
+        fetch_member = data_set.is_member(src)
         if fetch_member:
             member_name = src[src.find("(") + 1: src.find(")")]
         src = self._connection._shell.join_path(src)
         src = self._remote_expand_user(src)
 
+        # ********************************************************** #
+        #                Execute module on remote host               #
+        # ********************************************************** #
+        new_module_args = self._task.args.copy()
+        encoding_to = None
+        if encoding:
+            encoding_to = encoding.get("to", None)
+        if encoding is None or encoding_to is None:
+            new_module_args.update(
+                dict(encoding=dict(to=encode.Defaults.get_default_system_charset()))
+            )
+        remote_path = None
+
+        try:
+            fetch_res = self._execute_module(
+                module_name="ibm.ibm_zos_core.zos_fetch",
+                module_args=new_module_args,
+                task_vars=task_vars
+            )
+            ds_type = fetch_res.get("ds_type")
+            src = fetch_res.get("file")
+            remote_path = fetch_res.get("remote_path")
+
+            if fetch_res.get("msg"):
+                result["msg"] = fetch_res.get("msg")
+                result["stdout"] = fetch_res.get("stdout") or fetch_res.get(
+                    "module_stdout"
+                )
+                result["stderr"] = fetch_res.get("stderr") or fetch_res.get(
+                    "module_stderr"
+                )
+                result["stdout_lines"] = fetch_res.get("stdout_lines")
+                result["stderr_lines"] = fetch_res.get("stderr_lines")
+                result["rc"] = fetch_res.get("rc")
+                result["failed"] = True
+                return result
+
+            elif fetch_res.get("note"):
+                result["note"] = fetch_res.get("note")
+                return result
+
+        except Exception as err:
+            result["msg"] = "Failure during module execution"
+            result["stderr"] = str(err)
+            result["stderr_lines"] = str(err).splitlines()
+            result["failed"] = True
+            return result
+
         # ********************************************************** #
         #  Determine destination path:                               #
         #  1. If the 'flat' parameter is 'false', then hostname      #
@@ -216,52 +276,22 @@ def run(self, tmp=None, task_vars=None):
         local_checksum = _get_file_checksum(dest)
 
         # ********************************************************** #
-        #                Execute module on remote host               #
+        # Fetch remote data.
         # ********************************************************** #
-        new_module_args = self._task.args.copy()
-        encoding_to = None
-        if encoding:
-            encoding_to = encoding.get("to", None)
-        if encoding is None or encoding_to is None:
-            new_module_args.update(
-                dict(encoding=dict(to=encode.Defaults.get_default_system_charset()))
-            )
-        remote_path = None
         try:
-            fetch_res = self._execute_module(
-                module_name="ibm.ibm_zos_core.zos_fetch",
-                module_args=new_module_args,
-                task_vars=task_vars
-            )
-            ds_type = fetch_res.get("ds_type")
-            src = fetch_res.get("file")
-            remote_path = fetch_res.get("remote_path")
-
-            if fetch_res.get("msg"):
-                result["msg"] = fetch_res.get("msg")
-                result["stdout"] = fetch_res.get("stdout") or fetch_res.get(
-                    "module_stdout"
-                )
-                result["stderr"] = fetch_res.get("stderr") or fetch_res.get(
-                    "module_stderr"
-                )
-                result["stdout_lines"] = fetch_res.get("stdout_lines")
-                result["stderr_lines"] = fetch_res.get("stderr_lines")
-                result["rc"] = fetch_res.get("rc")
-                result["failed"] = True
-                return result
-
-            elif fetch_res.get("note"):
-                result["note"] = fetch_res.get("note")
-                return result
-
             if ds_type in SUPPORTED_DS_TYPES:
                 if ds_type == "PO" and os.path.isfile(dest) and not fetch_member:
                     result[
                         "msg"
                     ] = "Destination must be a directory to fetch a partitioned data set"
                     result["failed"] = True
                     return result
+                if ds_type == "GDG" and os.path.isfile(dest):
+                    result[
+                        "msg"
+                    ] = "Destination must be a directory to fetch a generation data group"
+                    result["failed"] = True
+                    return result
 
                 fetch_content = self._transfer_remote_content(
                     dest,
@@ -272,7 +302,7 @@ def run(self, tmp=None, task_vars=None):
                 if fetch_content.get("msg"):
                     return fetch_content
 
-                if validate_checksum and ds_type != "PO" and not is_binary:
+                if validate_checksum and ds_type != "GDG" and ds_type != "PO" and not is_binary:
                     new_checksum = _get_file_checksum(dest)
                     result["changed"] = local_checksum != new_checksum
                     result["checksum"] = new_checksum
@@ -286,6 +316,7 @@ def run(self, tmp=None, task_vars=None):
                 )
                 result["failed"] = True
                 return result
+
         except Exception as err:
             result["msg"] = "Failure during module execution"
             result["stderr"] = str(err)
@@ -311,7 +342,7 @@ def _transfer_remote_content(
         result = dict()
         _sftp_action = 'get'
 
-        if src_type == "PO":
+        if src_type == "PO" or src_type == "GDG":
             _sftp_action += ' -r'    # add '-r` to clone the source trees
 
         # To support multiple Ansible versions we must do some version detection and act accordingly
@@ -404,6 +435,6 @@ def _remote_cleanup(self, remote_path, src_type, encoding):
         # do not remove the original file.
         if not (src_type == "USS" and not encoding):
             rm_cmd = "rm -r {0}".format(remote_path)
-            if src_type != "PO":
+            if src_type != "PO" and src_type != "GDG":
                 rm_cmd = rm_cmd.replace(" -r", "")
             self._connection.exec_command(rm_cmd)

plugins/module_utils/data_set.py

@@ -540,8 +540,8 @@ def data_set_type(name, volume=None):
             volume (str) -- The volume the data set may reside on.
 
         Returns:
-            str -- The type of the data set (one of "PS", "PO", "DA", "KSDS",
-                    "ESDS", "LDS" or "RRDS").
+            str -- The type of the data set (one of "PS", "PO", "DA", "GDG",
+                    "KSDS", "ESDS", "LDS" or "RRDS").
             None -- If the data set does not exist or ZOAU is not able to determine
                     the type.
         """
@@ -551,10 +551,15 @@ def data_set_type(name, volume=None):
         data_sets_found = datasets.list_datasets(name)
 
         # Using the organization property when it's a sequential or partitioned
-        # dataset. VSAMs are not found by datasets.list_datasets.
+        # dataset. VSAMs and GDGs are not found by datasets.list_datasets.
         if len(data_sets_found) > 0:
             return data_sets_found[0].organization
 
+        # Now trying to list GDGs through gdgs.
+        data_sets_found = gdgs.list_gdg_names(name)
+        if len(data_sets_found) > 0:
+            return "GDG"
+
         # Next, trying to get the DATA information of a VSAM through
         # LISTCAT.
         output = DataSet._get_listcat_data(name)