[1.5.0-beta.1] [Bug] [zos_copy] Ported bugfix for newlines in record length (#639)

* Ported bugfix for #599

* Added changelog from 1.4.1

* Too many blanks lines

Signed-off-by: ddimatos <[email protected]>

---------

Signed-off-by: ddimatos <[email protected]>
Co-authored-by: ddimatos <[email protected]>

Author: rexemin

changelogs/fragments/599-copy-carriage-return.yml

@@ -0,0 +1,4 @@
+bugfixes:
+- zos_copy - fixes a bug where the computed record length for a new destination
+  dataset would include newline characters.
+  (https://github.com/ansible-collections/ibm_zos_core/pull/620)

plugins/module_utils/encode.py

@@ -453,6 +453,32 @@ def mvs_convert_encoding(
 
         return convert_rc
 
+    def uss_file_tag(self, file_path):
+        """Returns the current tag set for a file.
+        Arguments:
+            file_path {str} -- USS path to the file.
+        Returns:
+            str -- Current tag set for the file, as returned by 'ls -T'
+            None -- If the file does not exist or the command fails.
+        """
+        if not os.path.exists(file_path):
+            return None
+
+        try:
+            tag_cmd = "ls -T {0}".format(file_path)
+            rc, stdout, stderr = self.module.run_command(tag_cmd)
+
+            if rc != 0:
+                return None
+
+            # The output from 'ls -T' should be like this:
+            # t IBM-037     T=on  ansible-zos-copy-payload-D230123-T123818
+            # The second item from the split should be the tag.
+            ls_parts = stdout.split()
+            return ls_parts[1]
+        except Exception as err:
+            return None
+
 
 class EncodeError(Exception):
     def __init__(self, message):

plugins/modules/zos_copy.py

@@ -732,7 +732,6 @@ def copy_to_seq(
                                transferred data to
             conv_path {str} -- Path to the converted source file
             dest {str} -- Name of destination data set
-            src_ds_type {str} -- The type of source
         """
         new_src = conv_path or temp_path or src
         copy_args = dict()
@@ -916,6 +915,58 @@ def _merge_hash(self, *args):
             result.update(arg)
         return result
 
+    def file_has_crlf_endings(self, src):
+        """Reads src as a binary file and checks whether it uses CRLF or LF
+        line endings.
+
+        Arguments:
+            src {str} -- Path to a USS file
+
+        Returns:
+            {bool} -- True if the file uses CRLF endings, False if it uses LF
+                      ones.
+        """
+        with open(src, "rb") as src_file:
+            # readline() will read until it finds a \n.
+            content = src_file.readline()
+
+        # In EBCDIC, \r\n are bytes 0d and 15, respectively.
+        if content.endswith(b'\x0d\x15'):
+            return True
+        else:
+            return False
+
+    def create_temp_with_lf_endings(self, src):
+        """Creates a temporary file with the same content as src but without
+        carriage returns.
+
+        Arguments:
+            src {str} -- Path to a USS source file.
+
+        Raises:
+            CopyOperationError: If the conversion fails.
+
+        Returns:
+            {str} -- Path to the temporary file created.
+        """
+        try:
+            fd, converted_src = tempfile.mkstemp()
+            os.close(fd)
+
+            with open(converted_src, "wb") as converted_file:
+                with open(src, "rb") as src_file:
+                    current_line = src_file.read()
+                    converted_file.write(current_line.replace(b'\x0d', b''))
+
+            self._tag_file_encoding(converted_src, encode.Defaults.DEFAULT_EBCDIC_MVS_CHARSET)
+
+            return converted_src
+        except Exception as err:
+            raise CopyOperationError(
+                msg="Error while trying to convert EOL sequence for source.",
+                stderr=to_native(err)
+            )
+
 
 class USSCopyHandler(CopyHandler):
     def __init__(
@@ -1407,8 +1458,10 @@ def get_file_record_length(file):
         current_line = src_file.readline()
 
         while current_line:
-            if len(current_line) > max_line_length:
-                max_line_length = len(current_line)
+            line_length = len(current_line.rstrip("\n\r"))
+
+            if line_length > max_line_length:
+                max_line_length = line_length
 
             current_line = src_file.readline()
 
@@ -2316,6 +2369,37 @@ def run_module(module, arg_def):
         # Copy to sequential data set (PS / SEQ)
         # ---------------------------------------------------------------------
         elif dest_ds_type in data_set.DataSet.MVS_SEQ:
+            if src_ds_type == "USS" and not is_binary:
+                # Before copying into the destination dataset, we'll make sure that
+                # the source file doesn't contain any carriage returns that would
+                # result in empty records in the destination.
+                # Due to the differences between encodings, we'll normalize to IBM-037
+                # before checking the EOL sequence.
+                new_src = conv_path or temp_path or src
+                enc_utils = encode.EncodeUtils()
+                src_tag = enc_utils.uss_file_tag(new_src)
+
+                if src_tag == "untagged":
+                    src_tag = encode.Defaults.DEFAULT_EBCDIC_USS_CHARSET
+
+                if src_tag not in encode.Defaults.DEFAULT_EBCDIC_MVS_CHARSET:
+                    fd, converted_src = tempfile.mkstemp()
+                    os.close(fd)
+
+                    enc_utils.uss_convert_encoding(
+                        new_src,
+                        converted_src,
+                        src_tag,
+                        encode.Defaults.DEFAULT_EBCDIC_MVS_CHARSET
+                    )
+                    copy_handler._tag_file_encoding(converted_src, encode.Defaults.DEFAULT_EBCDIC_MVS_CHARSET)
+                    new_src = converted_src
+
+                if copy_handler.file_has_crlf_endings(new_src):
+                    new_src = copy_handler.create_temp_with_lf_endings(new_src)
+
+                conv_path = new_src
+
             copy_handler.copy_to_seq(
                 src,
                 temp_path,

tests/functional/modules/test_zos_copy_func.py

@@ -927,6 +927,55 @@ def test_copy_non_existent_file_fails(ansible_zos_module, is_remote):
         assert "does not exist" in result.get("msg")
 
 
+@pytest.mark.uss
+@pytest.mark.seq
+def test_copy_file_record_length_to_sequential_data_set(ansible_zos_module):
+    hosts = ansible_zos_module
+    dest = "USER.TEST.SEQ.FUNCTEST"
+
+    fd, src = tempfile.mkstemp()
+    os.close(fd)
+    with open(src, "w") as infile:
+        infile.write(DUMMY_DATA)
+
+    try:
+        hosts.all.zos_data_set(name=dest, state="absent")
+
+        copy_result = hosts.all.zos_copy(
+            src=src,
+            dest=dest,
+            remote_src=False,
+            is_binary=False
+        )
+
+        verify_copy = hosts.all.shell(
+            cmd="cat \"//'{0}'\" > /dev/null 2>/dev/null".format(dest),
+            executable=SHELL_EXECUTABLE,
+        )
+
+        verify_recl = hosts.all.shell(
+            cmd="dls -l {0}".format(dest),
+            executable=SHELL_EXECUTABLE,
+        )
+
+        for cp_res in copy_result.contacted.values():
+            assert cp_res.get("msg") is None
+            assert cp_res.get("changed") is True
+            assert cp_res.get("dest") == dest
+        for v_cp in verify_copy.contacted.values():
+            assert v_cp.get("rc") == 0
+        for v_recl in verify_recl.contacted.values():
+            assert v_recl.get("rc") == 0
+            stdout = v_recl.get("stdout").split()
+            assert len(stdout) == 5
+            assert stdout[1] == "PS"
+            assert stdout[2] == "FB"
+            assert stdout[3] == "31"
+    finally:
+        hosts.all.zos_data_set(name=dest, state="absent")
+        os.remove(src)
+
+
 @pytest.mark.uss
 @pytest.mark.seq
 @pytest.mark.parametrize("src", [
@@ -964,11 +1013,6 @@ def test_copy_file_to_non_existing_sequential_data_set(ansible_zos_module, src):
     finally:
         hosts.all.zos_data_set(name=dest, state="absent")
 
-        if src["is_file"]:
-            copy_result = hosts.all.zos_copy(src=src["src"], dest=dest, remote_src=src["is_remote"], is_binary=src["is_binary"])
-        else:
-            copy_result = hosts.all.zos_copy(content=src["src"], dest=dest, remote_src=src["is_remote"], is_binary=src["is_binary"])
-
 
 @pytest.mark.uss
 @pytest.mark.seq