make sure we copy the entire file when a file-like object is passed to write_file + add comments w.r.t. passing file handle in download_file

Author: boegel

easybuild/tools/filetools.py

@@ -246,17 +246,19 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over
     # cfr. https://docs.python.org/3/library/functions.html#open
     mode = 'a' if append else 'w'
 
-    is_file_like = hasattr(data, 'read')
+    data_is_file_obj = all(hasattr(data, x) for x in ('read', 'seek', 'tell', 'write'))
 
     # special care must be taken with binary data in Python 3
-    if sys.version_info[0] >= 3 and (isinstance(data, bytes) or is_file_like):
+    if sys.version_info[0] >= 3 and (isinstance(data, bytes) or data_is_file_obj):
         mode += 'b'
 
     # note: we can't use try-except-finally, because Python 2.4 doesn't support it as a single block
     try:
         mkdir(os.path.dirname(path), parents=True)
         with open_file(path, mode) as fh:
-            if is_file_like:
+            if data_is_file_obj:
+                # if a file-like object was provided, use copyfileobj (which reads the file in chunks)
+                data.seek(0)
                 shutil.copyfileobj(data, fh)
             else:
                 fh.write(data)
@@ -715,6 +717,10 @@ def download_file(filename, url, path, forced=False):
                 url_fd = response.raw
                 url_fd.decode_content = True
             _log.debug('response code for given url %s: %s' % (url, status_code))
+            # note: we pass the file object to write_file rather than reading the file first,
+            # to ensure the data is read in chunks (which prevents problems in Python 3.9+);
+            # cfr. https://github.com/easybuilders/easybuild-framework/issues/3455
+            # and https://bugs.python.org/issue42853
             write_file(path, url_fd, forced=forced, backup=True)
             _log.info("Downloaded file %s from url %s to %s" % (filename, url, path))
             downloaded = True

test/framework/filetools.py

@@ -705,8 +705,8 @@ def test_read_write_file(self):
         # test use of 'mode' in read_file
         self.assertEqual(ft.read_file(foo, mode='rb'), b'bar')
 
-    def test_write_file_like(self):
-        """Test writing from a file handle directly"""
+    def test_write_file_obj(self):
+        """Test writing from a file-like object directly"""
         # Write a text file
         fp = os.path.join(self.test_prefix, 'test.txt')
         fp_out = os.path.join(self.test_prefix, 'test_out.txt')
@@ -716,6 +716,12 @@ def test_write_file_like(self):
             ft.write_file(fp_out, fh)
         self.assertEqual(ft.read_file(fp_out), ft.read_file(fp))
 
+        # works fine even if same data was already read through the provided file handle
+        with ft.open_file(fp, 'rb') as fh:
+            fh.read(4)
+            ft.write_file(fp_out, fh)
+        self.assertEqual(ft.read_file(fp_out), ft.read_file(fp))
+
         # Write a binary file
         fp = os.path.join(self.test_prefix, 'test.bin')
         fp_out = os.path.join(self.test_prefix, 'test_out.bin')