[Enabler] [zos_backup_restore] Add support for GDG and special characters (#1527)

* Added tests for gds

* Added changelog and docs

* Fixed issue

* fixed unit tests

* Added changlog for zos_backup

* Reverted repr

---------

Co-authored-by: André Marcel Gutiérrez Benítez <[email protected]>

Author: fernandofloresg

changelogs/fragments/1527-zos_backup-gdg.yml

@@ -0,0 +1,5 @@
+minor_changes:
+  - zos_data_set - Added support for GDS relative name notation to include or exclude data sets when
+    operation is backup. Added support for data set names with special characters
+    like $, /#, and @.
+    (https://github.com/ansible-collections/ibm_zos_core/pull/1527).

plugins/modules/zos_backup_restore.py

@@ -51,6 +51,7 @@
         description:
           - When I(operation=backup), specifies a list of data sets or data set patterns
             to include in the backup.
+          - When I(operation=backup) GDS relative names are supported.
           - When I(operation=restore), specifies a list of data sets or data set patterns
             to include when restoring from a backup.
           - The single asterisk, C(*), is used in place of exactly one qualifier.
@@ -68,6 +69,7 @@
         description:
           - When I(operation=backup), specifies a list of data sets or data set patterns
             to exclude from the backup.
+          - When I(operation=backup) GDS relative names are supported.
           - When I(operation=restore), specifies a list of data sets or data set patterns
             to exclude when restoring from a backup.
           - The single asterisk, C(*), is used in place of exactly one qualifier.
@@ -117,6 +119,7 @@
       - There are no enforced conventions for backup names.
         However, using a common extension like C(.dzp) for UNIX files and C(.DZP) for data sets will
         improve readability.
+      - GDS relative names are supported when I(operation=restore).
     type: str
     required: True
   recover:
@@ -217,6 +220,15 @@
       exclude: user.private.*
     backup_name: MY.BACKUP.DZP
 
+- name: Backup a list of GDDs to data set my.backup.dzp
+  zos_backup_restore:
+    operation: backup
+    data_sets:
+      include:
+        - user.gdg(-1)
+        - user.gdg(0)
+    backup_name: my.backup.dzp
+
 - name: Backup all datasets matching the pattern USER.** to UNIX file /tmp/temp_backup.dzp, ignore recoverable errors.
   zos_backup_restore:
     operation: backup
@@ -312,18 +324,18 @@
     sms_management_class: DB2SMS10
 """
 
-from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.better_arg_parser import (
-    BetterArgParser,
-)
-from ansible.module_utils.basic import AnsibleModule
+import traceback
+from os import path
+from re import IGNORECASE, match, search
 
-from re import match, search, IGNORECASE
+from ansible.module_utils.basic import AnsibleModule
+from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.better_arg_parser import \
+    BetterArgParser
+from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.data_set import \
+    DataSet
+from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.import_handler import \
+    ZOAUImportError
 
-from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.import_handler import (
-    ZOAUImportError,
-)
-from os import path
-import traceback
 try:
     from zoautil_py import datasets
     from zoautil_py import exceptions as zoau_exceptions
@@ -386,8 +398,8 @@ def main():
         if operation == "backup":
             backup(
                 backup_name=backup_name,
-                include_data_sets=data_sets.get("include"),
-                exclude_data_sets=data_sets.get("exclude"),
+                include_data_sets=resolve_gds_name_if_any(data_sets.get("include")),
+                exclude_data_sets=resolve_gds_name_if_any(data_sets.get("exclude")),
                 volume=volume,
                 full_volume=full_volume,
                 temp_volume=temp_volume,
@@ -423,6 +435,26 @@ def main():
     module.exit_json(**result)
 
 
+def resolve_gds_name_if_any(data_set_list):
+    """Resolve all gds names in a list, if no gds relative name is found then
+    the original name will be kept.
+    Parameters
+    ----------
+    data_set_list : list
+        List of data set names.
+
+    Returns
+    -------
+    list
+        List of data set names with resolved gds names.
+    """
+    if isinstance(data_set_list, list):
+        for index, name in enumerate(data_set_list):
+            if DataSet.is_gds_relative_name(name):
+                data_set_list[index] = DataSet.resolve_gds_absolute_name(name)
+    return data_set_list
+
+
 def parse_and_validate_args(params):
     """Parse and validate arguments to be used by remainder of module.
 
@@ -662,7 +694,7 @@ def data_set_pattern_type(contents, dependencies):
         )
     for pattern in contents:
         if not match(
-            r"^(?:(?:[A-Za-z$#@\?\*]{1}[A-Za-z0-9$#@\-\?\*]{0,7})(?:[.]{1})){1,21}[A-Za-z$#@\*\?]{1}[A-Za-z0-9$#@\-\*\?]{0,7}$",
+            r"^(?:(?:[A-Za-z$#@\?\*]{1}[A-Za-z0-9$#@\-\?\*]{0,7})(?:[.]{1})){1,21}[A-Za-z$#@\*\?]{1}[A-Za-z0-9$#@\-\*\?]{0,7}(?:\(([-+]?[0-9]+)\)){0,1}$",
             str(pattern),
             IGNORECASE,
         ):
@@ -832,7 +864,7 @@ def backup_name_type(contents, dependencies):
     if contents is None:
         return None
     if not match(
-        r"^(?:(?:[A-Za-z$#@\?\*]{1}[A-Za-z0-9$#@\-\?\*]{0,7})(?:[.]{1})){1,21}[A-Za-z$#@\*\?]{1}[A-Za-z0-9$#@\-\*\?]{0,7}$",
+        r"^(?:(?:[A-Za-z$#@\?\*]{1}[A-Za-z0-9$#@\-\?\*]{0,7})(?:[.]{1})){1,21}[A-Za-z$#@\*\?]{1}[A-Za-z0-9$#@\-\*\?]{0,7}(?:\(([-+]?[0-9]+)\)){0,1}$",
         str(contents),
         IGNORECASE,
     ):

tests/functional/modules/test_zos_backup_restore.py

@@ -713,3 +713,85 @@ def test_restore_of_data_set_when_volume_does_not_exist(ansible_zos_module):
 #     finally:
 #         delete_data_set_or_file(hosts, data_set_name)
 #         delete_data_set_or_file(hosts, DATA_SET_BACKUP_LOCATION)
+
+
+@pytest.mark.parametrize("dstype", ["seq", "pds", "pdse"])
+def test_backup_gds(ansible_zos_module, dstype):
+    try:
+        hosts = ansible_zos_module
+        # We need to replace hyphens because of NAZARE-10614: dzip fails archiving data set names with '-'
+        data_set_name = get_tmp_ds_name(symbols=True).replace("-", "")
+        backup_dest = get_tmp_ds_name(symbols=True).replace("-", "")
+        results = hosts.all.zos_data_set(name=data_set_name, state="present", type="gdg", limit=3)
+        for result in results.contacted.values():
+            assert result.get("changed") is True
+            assert result.get("module_stderr") is None
+        results = hosts.all.zos_data_set(name=f"{data_set_name}(+1)", state="present", type=dstype)
+        for result in results.contacted.values():
+            assert result.get("changed") is True
+            assert result.get("module_stderr") is None
+        results = hosts.all.zos_data_set(name=f"{data_set_name}(+1)", state="present", type=dstype)
+        for result in results.contacted.values():
+            assert result.get("changed") is True
+            assert result.get("module_stderr") is None
+        results = hosts.all.zos_backup_restore(
+            operation="backup",
+            data_sets=dict(include=[f"{data_set_name}(-1)", f"{data_set_name}(0)"]),
+            backup_name=backup_dest,
+        )
+        for result in results.contacted.values():
+            assert result.get("changed") is True
+            assert result.get("module_stderr") is None
+    finally:
+        hosts.all.shell(cmd=f"drm ANSIBLE.* ")
+
+
+@pytest.mark.parametrize("dstype", ["seq", "pds", "pdse"])
+def test_backup_into_gds(ansible_zos_module, dstype):
+    """This test will create a dataset and backup it into a new generation of
+    backup data sets.
+    """
+    try:
+        hosts = ansible_zos_module
+        # We need to replace hyphens because of NAZARE-10614: dzip fails archiving data set names with '-'
+        data_set_name = get_tmp_ds_name(symbols=True).replace("-", "")
+        ds_name = get_tmp_ds_name(symbols=True).replace("-", "")
+        results = hosts.all.zos_data_set(name=data_set_name, state="present", type="gdg", limit=3)
+        for result in results.contacted.values():
+            assert result.get("changed") is True
+            assert result.get("module_stderr") is None
+        results = hosts.all.zos_data_set(name=f"{data_set_name}(+1)", state="present", type=dstype)
+        for result in results.contacted.values():
+            assert result.get("changed") is True
+            assert result.get("module_stderr") is None
+        results = hosts.all.zos_data_set(name=ds_name, state="present", type=dstype)
+        for result in results.contacted.values():
+            assert result.get("changed") is True
+            assert result.get("module_stderr") is None
+        ds_to_write = f"{ds_name}(MEM)" if dstype in ['pds', 'pdse'] else ds_name
+        results = hosts.all.shell(cmd=f"decho 'test line' \"{ds_to_write}\"")
+        for result in results.contacted.values():
+            assert result.get("changed") is True
+            assert result.get("module_stderr") is None
+        results = hosts.all.zos_backup_restore(
+            operation="backup",
+            data_sets=dict(include=[ds_name]),
+            backup_name=f"{data_set_name}.G0002V00",
+        )
+        for result in results.contacted.values():
+            assert result.get("changed") is True
+            assert result.get("module_stderr") is None
+        results = hosts.all.shell(cmd=f"drm \"{ds_name}\"")
+        for result in results.contacted.values():
+            assert result.get("changed") is True
+            assert result.get("module_stderr") is None
+        results = hosts.all.zos_backup_restore(
+            operation="restore",
+            backup_name=f"{data_set_name}(0)",
+        )
+        for result in results.contacted.values():
+            assert result.get("changed") is True
+            assert result.get("module_stderr") is None
+    finally:
+        hosts.all.shell(cmd=f"drm ANSIBLE.* ")
+