Adds option to skip encoding for zos_unarchive module (#2136)

* testing and adding skip_encoding

* changelog addition

* sanity fix

* review comments incorporation

Author: Rohitcodes28

changelogs/fragments/2136-zos_unarchive_skip_encoding_support.yml

@@ -0,0 +1,4 @@
+trivial:
+  - zos_unarchive - Adds support for skipping encoding in zos_unarchive module.
+    This allows users to skip encoding for certain files after unarchiving them.
+    (https://github.com/ansible-collections/ibm_zos_core/pull/2136)

plugins/modules/zos_unarchive.py

@@ -342,6 +342,12 @@
           - The destination I(dest) character set for the files to be written as.
         required: false
         type: str
+      skip_encoding:
+        description:
+          - List of names to skip encoding after unarchiving. This is only used if I(encoding) is set, otherwise is ignored.
+        required: false
+        type: list
+        elements: str
 attributes:
   action:
     support: full
@@ -419,6 +425,17 @@
     encoding:
       from: IBM-1047
       to: ISO8859-1
+
+- name: Encode the destination data set into Latin-1 after unarchiving.
+  zos_unarchive:
+    src: "USER.ARCHIVE.RESULT.TRS"
+    format:
+      name: terse
+    encoding:
+      from: IBM-1047
+      to: ISO8859-1
+      skip_encoding:
+        - USER.ARCHIVE.TEST1
 '''
 
 RETURN = r'''
@@ -453,7 +470,11 @@
       List of files or data sets that were failed while encoding.
     type: list
     returned: success
-
+skipped_encoding_targets:
+    description:
+      List of files or data sets that were skipped while encoding.
+    type: list
+    returned: success
 '''
 
 import abc
@@ -525,6 +546,8 @@ def __init__(self, module):
             The encoding of the source file.
         to_encoding: str
             The required encoding of the destination file.
+        skip_encoding : list[str]
+            List of paths to exclude in encoding.
         """
         self.module = module
         self.src = module.params.get("src")
@@ -547,6 +570,8 @@ def __init__(self, module):
             self.dest = os.path.dirname(self.src)
         self.encoded = list()
         self.failed_on_encoding = list()
+        self.skip_encoding = encoding_param.get("skip_encoding")
+        self.skipped_encoding_targets = list()
 
     @abc.abstractmethod
     def extract_src(self):
@@ -591,6 +616,17 @@ def update_permissions(self):
             file_args = self.module.load_file_common_arguments(self.module.params, path=file_name)
             self.module.set_fs_attributes_if_different(file_args, self.changed)
 
+    def encoding_targets(self):
+        """Finds encoding target files in host.
+        """
+        if self.skip_encoding:
+            self.encode_targets = [
+                path for path in self.targets if path not in self.skip_encoding
+            ]
+            self.skipped_encoding_targets = self.skip_encoding
+        else:
+            self.encode_targets = self.targets
+
     def encode_destination(self):
         """Convert encoding for given destination
         Returns
@@ -602,7 +638,7 @@ def encode_destination(self):
         self.encoded = []
         self.failed_on_encoding = []
 
-        for target in self.targets:
+        for target in self.encode_targets:
             try:
                 file_path = os.path.normpath(os.path.join(self.dest, target))
                 convert_rc = enc_utils.uss_convert_encoding_prev(
@@ -617,7 +653,8 @@ def encode_destination(self):
 
         return {
             "encoded": self.encoded,
-            "failed_on_encoding": self.failed_on_encoding
+            "failed_on_encoding": self.failed_on_encoding,
+            "skipped_encoding_targets": self.skipped_encoding_targets
         }
 
     @property
@@ -636,6 +673,7 @@ def result(self):
             'missing': self.missing,
             'encoded': getattr(self, 'encoded', []),
             'failed_on_encoding': getattr(self, 'failed_on_encoding', []),
+            'skipped_encoding_targets' : getattr(self, 'skipped_encoding_targets', []),
         }
 
     def extract_all(self, members):
@@ -1201,6 +1239,17 @@ def clean_environment(self, data_sets=None, uss_files=None, remove_targets=False
             for target in self.targets:
                 data_set.DataSet.ensure_absent(target)
 
+    def encoding_targets(self):
+        """Finds encoding target datasets in host.
+        """
+        if self.skip_encoding:
+            self.encode_targets = [
+                path for path in self.targets if path not in self.skip_encoding
+            ]
+            self.skipped_encoding_targets = self.skip_encoding
+        else:
+            self.encode_targets = self.targets
+
     def encode_destination(self):
         """Convert encoding for given destination
         Returns
@@ -1212,7 +1261,7 @@ def encode_destination(self):
         self.encoded = []
         self.failed_on_encoding = []
 
-        for target in self.targets:
+        for target in self.encode_targets:
             try:
                 ds_utils = data_set.DataSetUtils(target, tmphlq=self.tmphlq)
                 ds_type = ds_utils.ds_type()
@@ -1232,7 +1281,8 @@ def encode_destination(self):
                 self.failed_on_encoding.append(os.path.abspath(target))
         return {
             "encoded": self.encoded,
-            "failed_on_encoding": self.failed_on_encoding
+            "failed_on_encoding": self.failed_on_encoding,
+            "skipped_encoding_targets": self.skipped_encoding_targets
         }
 
 
@@ -1623,6 +1673,11 @@ def run_module():
                     "to": dict(
                         type='str',
                         required=False,
+                    ),
+                    "skip_encoding": dict(
+                        type='list',
+                        elements='str',
+                        required=False,
                     )
                 }
             ),
@@ -1706,7 +1761,8 @@ def run_module():
             type='dict',
             options={
                 'from' : dict(type='str'),
-                "to" : dict(type='str')
+                'to' : dict(type='str'),
+                'skip_encoding' : dict(type='list', elements='str', required=False),
             }
         ),
     )
@@ -1733,10 +1789,12 @@ def run_module():
 
     encoding = parsed_args.get("encoding")
     if unarchive.dest_unarchived() and encoding:
+        unarchive.encoding_targets()
         encoding_result = unarchive.encode_destination()
         unarchive.result.update({
             "encoded": encoding_result.get("encoded", []),
-            "failed_on_encoding": encoding_result.get("failed_on_encoding", [])
+            "failed_on_encoding": encoding_result.get("failed_on_encoding", []),
+            "skipped_encoding_targets": encoding_result.get("skipped_encoding_targets", [])
         })
 
     module.exit_json(**unarchive.result)

tests/functional/modules/test_zos_unarchive_func.py

@@ -142,6 +142,7 @@ def create_multiple_members(ansible_zos_module, pds_name, member_base_name, n):
 # - test_uss_unarchive_list
 # - test_uss_unarchive_copy_to_remote
 # - test_uss_unarchive_encoding
+# - test_uss_unarchive_encoding_skip_encoding
 
 # Core functionality tests
 # Test unarchive with no options
@@ -433,6 +434,55 @@ def test_uss_unarchive_encoding(ansible_zos_module, ds_format):
     finally:
         hosts.all.file(path=f"{USS_TEMP_DIR}", state="absent")
 
+
+@pytest.mark.uss
+@pytest.mark.parametrize("ds_format", USS_FORMATS)
+def test_uss_unarchive_encoding_skip_encoding(ansible_zos_module, ds_format):
+    try:
+        hosts = ansible_zos_module
+        hosts.all.file(path=f"{USS_TEMP_DIR}", state="absent")
+        hosts.all.file(path=USS_TEMP_DIR, state="directory")
+        set_uss_test_env(hosts, USS_TEST_FILES)
+        dest = f"{USS_TEMP_DIR}/archive.{ds_format}"
+        hosts.all.zos_archive(
+            src=list(USS_TEST_FILES.keys()),
+            dest=dest,
+            format={
+                "name":ds_format
+            }
+        )
+        # remove files
+        for file in USS_TEST_FILES.keys():
+            hosts.all.file(path=file, state="absent")
+        #specify encoding and skip_encoding file
+        first_file_to_skip = [next(iter(USS_TEST_FILES.keys()))]
+        encoding={
+            "from": TO_ENCODING,
+            "to": FROM_ENCODING,
+            "skip_encoding": first_file_to_skip
+        }
+        #unarchive files
+        unarchive_result = hosts.all.zos_unarchive(
+            src=dest,
+            format={
+                "name":ds_format
+            },
+            remote_src=True,
+            encoding= encoding,
+        )
+        hosts.all.shell(cmd=f"ls {USS_TEMP_DIR}")
+
+        for result in unarchive_result.contacted.values():
+            assert result.get("failed", False) is False
+            assert result.get("changed") is True
+            # Command to assert the file is in place
+            cmd_result = hosts.all.shell(cmd=f"ls {USS_TEMP_DIR}")
+            for c_result in cmd_result.contacted.values():
+                for file in USS_TEST_FILES.keys():
+                    assert file[len(USS_TEMP_DIR)+1:] in c_result.get("stdout")
+    finally:
+        hosts.all.file(path=f"{USS_TEMP_DIR}", state="absent")
+
 ######################################################################
 #
 #   MVS data sets tests
@@ -450,6 +500,7 @@ def test_uss_unarchive_encoding(ansible_zos_module, ds_format):
 # - test_mvs_unarchive_remote_src
 # - test_mvs_unarchive_encoding
 # - test_mvs_unarchive_multiple_data_set_use_adrdssu_encoding
+# - test_mvs_unarchive_encoding_skip_encoding
 
 
 @pytest.mark.ds
@@ -1434,6 +1485,136 @@ def test_mvs_unarchive_encoding(
         hosts.all.zos_data_set(name=mvs_dest_archive, state="absent")
 
 
+@pytest.mark.ds
+@pytest.mark.parametrize(
+    "ds_format", [
+        "terse",
+         "xmit"
+        ])
+@pytest.mark.parametrize(
+    "data_set", [
+        {
+            "dstype":"seq",
+            "members":[""]
+        },
+        {
+            "dstype":"pds",
+            "members":["MEM1", "MEM2"]
+        },
+        ]
+)
+@pytest.mark.parametrize(
+    "record_length", [80]
+)
+@pytest.mark.parametrize(
+    "encoding", [
+        {"from": "IBM-1047", "to": "ISO8859-1"},
+    ]
+)
+def test_mvs_unarchive_encoding_skip_encoding(
+    ansible_zos_module,
+    ds_format,
+    data_set,
+    record_length,
+    encoding
+):
+    try:
+        hosts = ansible_zos_module
+        mvs_dest_archive = get_tmp_ds_name()
+        dataset = get_tmp_ds_name(3)
+        hlq = "ANSIBLE"
+        record_format = "fb"
+        # Clean env
+        hosts.all.zos_data_set(name=mvs_dest_archive, state="absent")
+        # Create source data set
+        hosts.all.zos_data_set(
+            name=dataset,
+            type=data_set.get("dstype"),
+            state="present",
+            record_length=record_length,
+            record_format=record_format,
+            replace=True
+        )
+        # Create members if needed
+        if data_set.get("dstype") in ["pds", "pdse"]:
+            for member in data_set.get("members"):
+                hosts.all.zos_data_set(
+                    name=f"{dataset}({member})",
+                    type="member",
+                    state="present",
+                    replace=True
+                )
+        test_line = "a" * record_length
+        for member in data_set.get("members"):
+            if member == "":
+                ds_to_write = f"{dataset}"
+            else:
+                ds_to_write = f"{dataset}({member})"
+            hosts.all.shell(cmd=f"decho '{test_line}' \"{ds_to_write}\"")
+
+        format_dict = {
+            "name":ds_format
+        }
+        if ds_format == "terse":
+            format_dict["format_options"] = {
+                "terse_pack":"spack"
+            }
+        archive_result = hosts.all.zos_archive(
+            src=dataset,
+            dest=mvs_dest_archive,
+            format=format_dict,
+            dest_data_set={
+                "name":dataset,
+                "type":"seq",
+                "record_format":record_format,
+                "record_length":record_length
+            },
+        )
+        # assert response is positive
+        for result in archive_result.contacted.values():
+            assert result.get("changed") is True
+            assert result.get("dest") == mvs_dest_archive
+            assert dataset in result.get("archived")
+            cmd_result = hosts.all.shell(cmd = f"""dls "{hlq}.*" """)
+            for c_result in cmd_result.contacted.values():
+                assert mvs_dest_archive in c_result.get("stdout")
+
+        hosts.all.zos_data_set(name=dataset, state="absent")
+
+        if ds_format == "terse":
+            del format_dict["format_options"]["terse_pack"]
+
+        #skipping some files to encode 
+        skip_encoding_list = [dataset]
+        current_encoding_config = encoding.copy()
+        current_encoding_config["skip_encoding"] = skip_encoding_list
+
+        # Unarchive action
+        unarchive_result = hosts.all.zos_unarchive(
+            src=mvs_dest_archive,
+            format=format_dict,
+            remote_src=True,
+            dest_data_set={
+                "name":dataset,
+                "type":data_set.get("dstype"),
+                "record_format":record_format,
+                "record_length":record_length           
+            },
+            encoding=encoding,
+        )
+        # assert response is positive
+        for result in unarchive_result.contacted.values():
+            assert result.get("changed") is True
+            assert result.get("failed", False) is False
+            # assert result.get("dest") == mvs_dest_archive
+            # assert data_set.get("name") in result.get("archived")
+            cmd_result = hosts.all.shell(cmd = f"""dls "{hlq}.*" """)
+            for c_result in cmd_result.contacted.values():
+                assert dataset in c_result.get("stdout")
+    finally:
+        hosts.all.zos_data_set(name=dataset, state="absent")
+        hosts.all.zos_data_set(name=mvs_dest_archive, state="absent")
+
 
 @pytest.mark.ds
 @pytest.mark.parametrize(