[Enabler][zos_archive][zos_unarchive] Add support for GDGs and validate symbols support (#1511)

* Added test symbols

* Added tests for GDG and sepcial chars

* Added changelog

* Adding support for gds in zos_unarchive

* Added GDS resolve function into archive

* Added

* Added test, example and modified docs

* Added docs

* Added support for archive into a GDS

* Removed comments

* Fixed test

* Updated changelog

* Updated changelog

Author: fernandofloresg

changelogs/fragments/1511-zos_archive_unarchive-gdg-support.yml

@@ -0,0 +1,7 @@
+minor_changes:
+  - zos_archive - Added support for GDG and GDS relative name notation to archive data sets.
+    Added support for data set names with special characters like $, /#, /- and @.
+    (https://github.com/ansible-collections/ibm_zos_core/pull/1511).
+  - zos_unarchive - Added support for data set names with special characters
+    like $, /#, /- and @.
+    (https://github.com/ansible-collections/ibm_zos_core/pull/1511).

plugins/module_utils/data_set.py

@@ -1030,7 +1030,7 @@ def create(
         original_args = locals()
         formatted_args = DataSet._build_zoau_args(**original_args)
         try:
-            datasets.create(**formatted_args)
+            data_set = datasets.create(**formatted_args)
         except exceptions._ZOAUExtendableException as create_exception:
             raise DatasetCreateError(
                 raw_name if raw_name else name,
@@ -1046,9 +1046,8 @@ def create(
                 raw_name if raw_name else name,
                 msg="Unable to verify the data set was created. Received DatasetVerificationError from ZOAU.",
             )
-        # With ZOAU 1.3 we switched from getting a ZOAUResponse obj to a Dataset obj, previously we returned
-        # response.rc now we just return 0 if nothing failed
-        return 0
+        changed = data_set is not None
+        return changed
 
     @staticmethod
     def delete(name):
@@ -1896,7 +1895,7 @@ def __init__(
             # with ZOAU
             self.record_format = None
 
-    def create(self):
+    def create(self, tmp_hlq=None, replace=True, force=False):
         """Creates the data set in question.
 
         Returns
@@ -1907,7 +1906,6 @@ def create(self):
         arguments = {
             "name" : self.name,
             "raw_name" : self.raw_name,
-            "replace" : self.replace,
             "type" : self.data_set_type,
             "space_primary" : self.space_primary,
             "space_secondary" : self.space_secondary,
@@ -1922,11 +1920,20 @@ def create(self):
             "sms_data_class" : self.sms_data_class,
             "sms_management_class" : self.sms_management_class,
             "volumes" : self.volumes,
-            "tmp_hlq" : self.tmp_hlq,
-            "force" : self.force,
+            "tmp_hlq" : tmp_hlq,
+            "force" : force,
         }
-        DataSet.create(**arguments)
-        self.set_state("present")
+        formatted_args = DataSet._build_zoau_args(**arguments)
+        changed = False
+        if DataSet.data_set_exists(self.name):
+            DataSet.delete(self.name)
+            changed = True
+        zoau_data_set = datasets.create(**formatted_args)
+        if zoau_data_set is not None:
+            self.set_state("present")
+            self.name = zoau_data_set.name
+            return True
+        return changed
 
     def ensure_present(self, tmp_hlq=None, replace=False, force=False):
         """ Make sure that the data set is created or fail creating it.

plugins/modules/zos_archive.py

@@ -12,7 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import (absolute_import, division, print_function)
+from __future__ import absolute_import, division, print_function
+
 __metaclass__ = type
 
 DOCUMENTATION = r'''
@@ -36,6 +37,7 @@
       - List of names or globs of UNIX System Services (USS) files,
         PS (sequential data sets), PDS, PDSE to compress or archive.
       - USS file paths should be absolute paths.
+      - GDS relative notation is supported.
       - "MVS data sets supported types are: C(SEQ), C(PDS), C(PDSE)."
       - VSAMs are not supported.
     type: list
@@ -123,9 +125,9 @@
     required: true
   exclude:
     description:
-      - Remote absolute path, glob, or list of paths, globs or data set name
-        patterns for the file, files or data sets to exclude from src list
-        and glob expansion.
+      - Remote absolute path, glob, or list of paths, globs, data set name
+        patterns or generation data sets (GDSs) in relative notation for the file,
+        files or data sets to exclude from src list and glob expansion.
       - "Patterns (wildcards) can contain one of the following, `?`, `*`."
       - "* matches everything."
       - "? matches any single character."
@@ -331,7 +333,7 @@
       name: tar
 
 # Archive multiple files
-- name: Compress list of files into a zip
+- name: Archive list of files into a zip
   zos_archive:
     src:
       - /tmp/archive/foo.txt
@@ -341,15 +343,15 @@
     name: zip
 
 # Archive one data set into terse
-- name: Compress data set into a terse
+- name: Archive data set into a terse
   zos_archive:
     src: "USER.ARCHIVE.TEST"
     dest: "USER.ARCHIVE.RESULT.TRS"
     format:
       name: terse
 
 # Use terse with different options
-- name: Compress data set into a terse, specify pack algorithm and use adrdssu
+- name: Archive data set into a terse, specify pack algorithm and use adrdssu
   zos_archive:
     src: "USER.ARCHIVE.TEST"
     dest: "USER.ARCHIVE.RESULT.TRS"
@@ -360,13 +362,34 @@
         use_adrdssu: true
 
 # Use a pattern to store
-- name: Compress data set pattern using xmit
+- name: Archive data set pattern using xmit
   zos_archive:
     src: "USER.ARCHIVE.*"
     exclude_sources: "USER.ARCHIVE.EXCLUDE.*"
     dest: "USER.ARCHIVE.RESULT.XMIT"
     format:
       name: xmit
+
+- name: Archive multiple GDSs into a terse
+  zos_archive:
+    src:
+      - "USER.GDG(0)"
+      - "USER.GDG(-1)"
+      - "USER.GDG(-2)"
+    dest: "USER.ARCHIVE.RESULT.TRS"
+    format:
+      name: terse
+      format_options:
+        use_adrdssu: True
+
+- name: Archive multiple data sets into a new GDS
+  zos_archive:
+    src: "USER.ARCHIVE.*"
+    dest: "USER.GDG(+1)"
+    format:
+      name: terse
+      format_options:
+        use_adrdssu: True
 '''
 
 RETURN = r'''
@@ -415,27 +438,22 @@
     returned: always
 '''
 
-from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils._text import to_bytes
-from ansible_collections.ibm.ibm_zos_core.plugins.module_utils import (
-    better_arg_parser,
-    data_set,
-    validation,
-    mvs_cmd,
-)
-from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.import_handler import (
-    ZOAUImportError,
-)
-import os
-import tarfile
-import zipfile
 import abc
 import glob
-import re
 import math
+import os
+import re
+import tarfile
 import traceback
+import zipfile
 from hashlib import sha256
 
+from ansible.module_utils._text import to_bytes
+from ansible.module_utils.basic import AnsibleModule
+from ansible_collections.ibm.ibm_zos_core.plugins.module_utils import (
+    better_arg_parser, data_set, mvs_cmd, validation)
+from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.import_handler import \
+    ZOAUImportError
 
 try:
     from zoautil_py import datasets
@@ -1278,11 +1296,17 @@ def expand_mvs_paths(self, paths):
         """
         expanded_path = []
         for path in paths:
+            e_path = []
             if '*' in path:
                 # list_dataset_names returns a list of data set names or empty.
                 e_paths = datasets.list_dataset_names(path)
             else:
                 e_paths = [path]
+
+            # resolve GDS relative names
+            for index, e_path in enumerate(e_paths):
+                if data_set.DataSet.is_gds_relative_name(e_path):
+                    e_paths[index] = data_set.DataSet.resolve_gds_absolute_name(e_path)
             expanded_path.extend(e_paths)
         return expanded_path
 
@@ -1415,17 +1439,18 @@ def archive_targets(self):
                 self.module.fail_json(
                     msg="To archive multiple source data sets, you must use option 'use_adrdssu=True'.")
             source = self.targets[0]
-        # dest = self.create_dest_ds(self.dest)
-        dest, changed = self._create_dest_data_set(
+        dataset = data_set.MVSDataSet(
             name=self.dest,
-            replace=True,
-            type='seq',
+            data_set_type='seq',
             record_format='fb',
             record_length=AMATERSE_RECORD_LENGTH,
             space_primary=self.dest_data_set.get("space_primary"),
-            space_type=self.dest_data_set.get("space_type"))
+            space_type=self.dest_data_set.get("space_type")
+        )
+        changed = dataset.create(replace=True)
         self.changed = self.changed or changed
-        self.add(source, dest)
+        self.dest = dataset.name
+        self.add(source, self.dest)
         self.clean_environment(data_sets=self.tmp_data_sets)
 
 
@@ -1509,16 +1534,19 @@ def archive_targets(self):
                     msg="To archive multiple source data sets, you must use option 'use_adrdssu=True'.")
             source = self.sources[0]
         # dest = self.create_dest_ds(self.dest)
-        dest, changed = self._create_dest_data_set(
+        dataset = data_set.MVSDataSet(
             name=self.dest,
-            replace=True,
-            type='seq',
+            data_set_type='seq',
             record_format='fb',
             record_length=XMIT_RECORD_LENGTH,
             space_primary=self.dest_data_set.get("space_primary"),
-            space_type=self.dest_data_set.get("space_type"))
+            space_type=self.dest_data_set.get("space_type")
+        )
+        changed = dataset.create(replace=True)
+        self.changed = self.changed or changed
         self.changed = self.changed or changed
-        self.add(source, dest)
+        self.dest = dataset.name
+        self.add(source, self.dest)
         self.clean_environment(data_sets=self.tmp_data_sets)
 
     def get_error_hint(self, output):

plugins/modules/zos_unarchive.py

@@ -35,7 +35,7 @@
       - The remote absolute path or data set of the archive to be uncompressed.
       - I(src) can be a USS file or MVS data set name.
       - USS file paths should be absolute paths.
-      - MVS data sets supported types are C(SEQ), C(PDS), C(PDSE).
+      - GDS relative names are supported C(e.g. USER.GDG(-1)).
     type: str
     required: true
   format:
@@ -145,6 +145,7 @@
     description:
       - A list of directories, files or data set names to extract from the
         archive.
+      - GDS relative names are supported C(e.g. USER.GDG(-1)).
       - When C(include) is set, only those files will we be extracted leaving
         the remaining files in the archive.
       - Mutually exclusive with exclude.
@@ -155,6 +156,7 @@
     description:
       - List the directory and file or data set names that you would like to
         exclude from the unarchive action.
+      - GDS relative names are supported C(e.g. USER.GDG(-1)).
       - Mutually exclusive with include.
     type: list
     elements: str
@@ -349,6 +351,13 @@
       - USER.ARCHIVE.TEST1
       - USER.ARCHIVE.TEST2
 
+# Unarchive a GDS
+- name: Unarchive a terse data set and excluding data sets from unpacking.
+  zos_unarchive:
+    src: "USER.ARCHIVE(0)"
+    format:
+      name: terse
+
 # List option
 - name: List content from XMIT
   zos_unarchive:
@@ -623,6 +632,8 @@ def __init__(self, module):
         self.dest_data_set = module.params.get("dest_data_set")
         self.dest_data_set = dict() if self.dest_data_set is None else self.dest_data_set
         self.source_size = 0
+        if data_set.DataSet.is_gds_relative_name(self.src):
+            self.src = data_set.DataSet.resolve_gds_absolute_name(self.src)
 
     def dest_type(self):
         return "MVS"
@@ -709,14 +720,14 @@ def _create_dest_data_set(
     def _get_include_data_sets_cmd(self):
         include_cmd = "INCL( "
         for include_ds in self.include:
-            include_cmd += " '{0}', - \n".format(include_ds)
+            include_cmd += " '{0}', - \n".format(include_ds.upper())
         include_cmd += " ) - \n"
         return include_cmd
 
     def _get_exclude_data_sets_cmd(self):
         exclude_cmd = "EXCL( - \n"
         for exclude_ds in self.exclude:
-            exclude_cmd += " '{0}', - \n".format(exclude_ds)
+            exclude_cmd += " '{0}', - \n".format(exclude_ds.upper())
         exclude_cmd += " ) - \n"
         return exclude_cmd
 
@@ -1143,13 +1154,13 @@ def run_module():
         module.fail_json(msg="Parameter verification failed", stderr=str(err))
     unarchive = get_unarchive_handler(module)
 
+    if not unarchive.src_exists():
+        module.fail_json(msg="{0} does not exists, please provide a valid src.".format(module.params.get("src")))
+
     if unarchive.list:
         unarchive.list_archive_content()
         module.exit_json(**unarchive.result)
 
-    if not unarchive.src_exists():
-        module.fail_json(msg="{0} does not exists, please provide a valid src.".format(module.params.get("src")))
-
     unarchive.extract_src()
 
     if unarchive.dest_unarchived() and unarchive.dest_type() == "USS":