[zos_script][zos_copy] Support automatic removal of carriage return characters (CR) when copying files (#1954)

* Changes related to removing carriage return from file while copying to USS

* Update zos_copy.py

* Update zos_copy.py

* Adding new method just to remove CR

* Update zos_copy.py

* Update test_zos_copy_func.py

* Create 1954-zos_copy-cr-removal-while-copying-file-to-remote.yml

* Updating comment

* Resolving PR comments

* Updating chunk size to 32 MB

* Update zos_copy.py

* adding logging for lock testcase

* Update test_zos_copy_func.py

* Update zos_copy.py

* removing extra method

* Update zos_copy.py

* Fixed sanity issues

* Updated script

---------

Co-authored-by: Fernando Flores <[email protected]>

Author: surendrababuravella

changelogs/fragments/1954-zos_copy-cr-removal-while-copying-file-to-remote.yml

@@ -0,0 +1,7 @@
+minor_changes:
+   - zos_script - Support automatic removal of carriage return line breaks [CR, CRLF] when copying local files to USS.
+     (https://github.com/ansible-collections/ibm_zos_core/pull/1954).
+bugfixes:
+   - zos_copy - the carriage return characters were being removed from only first 1024 bytes of a file. Now fixed that issue to support 
+     removal of the carriage return characters from the complete file content if the file size is more than 1024 bytes.
+      (https://github.com/ansible-collections/ibm_zos_core/pull/1954).

plugins/modules/zos_copy.py

@@ -1437,8 +1437,8 @@ def file_has_crlf_endings(self, src):
             content = src_file.read(1024)
 
             while content:
-                # In EBCDIC, \r\n are bytes 0d and 15, respectively.
-                if b'\x0d\x15' in content:
+                # In EBCDIC, \r is bytes 0d
+                if b'\x0d' in content:
                     return True
                 content = src_file.read(1024)
 
@@ -1466,12 +1466,15 @@ def create_temp_with_lf_endings(self, src):
         try:
             fd, converted_src = tempfile.mkstemp(dir=os.environ['TMPDIR'])
             os.close(fd)
-
+            # defining 32 MB chunk size for reading large files efficiently
+            chunk_size = 32 * 1024 * 1024
             with open(converted_src, "wb") as converted_file:
                 with open(src, "rb") as src_file:
-                    chunk = src_file.read(1024)
-                    # In IBM-037, \r is the byte 0d.
-                    converted_file.write(chunk.replace(b'\x0d', b''))
+                    chunk = src_file.read(chunk_size)
+                    while chunk:
+                        # In IBM-037, \r is the byte 0d.
+                        converted_file.write(chunk.replace(b'\x0d', b''))
+                        chunk = src_file.read(chunk_size)
 
             self._tag_file_encoding(converted_src, "IBM-037")
 
@@ -1482,6 +1485,45 @@ def create_temp_with_lf_endings(self, src):
                 stderr=to_native(err)
             )
 
+    def remove_cr_endings(self, src):
+        """Creates a temporary file with the same content as src but without
+        carriage returns.
+
+        Parameters
+        ----------
+        src : str
+            Path to a USS source file.
+
+        Returns
+        -------
+        str
+            Path to the temporary file created.
+
+        Raises
+        ------
+        CopyOperationError
+            If the conversion fails.
+        """
+        try:
+            fd, converted_src = tempfile.mkstemp(dir=os.environ['TMPDIR'])
+            os.close(fd)
+            # defining 32 MB chunk size for reading large files efficiently
+            chunk_size = 32 * 1024 * 1024
+            with open(converted_src, "wb") as converted_file:
+                with open(src, "rb") as src_file:
+                    chunk = src_file.read(chunk_size)
+                    while chunk:
+                        # In IBM-037, \r is the byte 0d.
+                        converted_file.write(chunk.replace(b'\x0d', b''))
+                        chunk = src_file.read(chunk_size)
+
+            return converted_src
+        except Exception as err:
+            raise CopyOperationError(
+                msg="Error while trying to convert EOL sequence for source.",
+                stderr=to_native(err)
+            )
+
 
 class USSCopyHandler(CopyHandler):
     def __init__(
@@ -3670,6 +3712,11 @@ def run_module(module, arg_def):
         # Copy to USS file or directory
         # ---------------------------------------------------------------------
         if is_uss:
+            # Removing the carriage return characters
+            if src_ds_type == "USS" and not is_binary and not executable:
+                new_src = conv_path or src
+                if os.path.isfile(new_src):
+                    conv_path = copy_handler.remove_cr_endings(new_src)
             uss_copy_handler = USSCopyHandler(
                 module,
                 is_binary=is_binary,

tests/functional/modules/test_zos_script_func.py

@@ -148,6 +148,20 @@ def create_local_file(content, suffix):
 
     return file_path
 
+def create_local_file_with_carriagereturn(content, suffix):
+    """Creates a tempfile that has the given content."""
+
+    fd, file_path = tempfile.mkstemp(
+        prefix='zos_script',
+        suffix=suffix
+    )
+    os.close(fd)
+
+    with open(file_path, 'w', encoding="utf-8", newline='\r\n') as f:
+        f.write(content)
+
+    return file_path
+
 
 def test_rexx_script_without_args(ansible_zos_module):
     hosts = ansible_zos_module
@@ -604,6 +618,37 @@ def test_rexx_script_with_args_remote_src(ansible_zos_module):
         hosts.all.file(path=script_path, state="absent")
 
 
+def test_rexx_script_with_args_and_carriagereturn(ansible_zos_module):
+    hosts = ansible_zos_module
+    script_path = ''
+    try:
+        rexx_script = REXX_SCRIPT_ARGS
+        script_path = create_local_file_with_carriagereturn(rexx_script, 'rexx')
+
+        first_arg = 'one'
+        second_arg = 'two'
+        args = f'FIRST={first_arg} SECOND={second_arg}'
+        cmd = f"{script_path} '{args}'"
+
+        zos_script_result = hosts.all.zos_script(
+            cmd=cmd
+        )
+
+        for result in zos_script_result.contacted.values():
+            assert result.get('changed') is True
+            assert result.get('failed', False) is False
+            assert result.get('rc') == 0
+            assert first_arg in result.get('stdout', '')
+            assert second_arg in result.get('stdout', '')
+            # Making sure the action plugin passed every argument to the module.
+            assert args in result.get('invocation').get('module_args').get('cmd')
+            assert args in result.get('remote_cmd')
+            assert result.get('stderr', '') == ''
+    finally:
+        if os.path.exists(script_path):
+            os.remove(script_path)
+
+
 def test_job_script_async(get_config):
     # Creating temp REXX file used by the playbook.
     try:
@@ -632,7 +677,7 @@ def test_job_script_async(get_config):
                 cut_python_path,
                 python_version,
                 script_path
-            )), 
+            )),
             playbook.name
         ))
 
@@ -642,7 +687,7 @@ def test_job_script_async(get_config):
                 ssh_key,
                 user,
                 python_path
-            )), 
+            )),
             inventory.name
         ))