feat: handle empty file streaming by yielding empty bytes

#fixes 222

Author: dnlbauer

rocrate/model/dataset.py

@@ -105,10 +105,16 @@ def _stream_folder_from_path(self, chunk_size=8192):
                 for name in files:
                     source = root / name
                     dest = source.relative_to(Path(path).parent)
+                    is_empty = True
                     with open(source, 'rb') as f:
                         while chunk := f.read(chunk_size):
+                            is_empty = False
                             yield str(dest), chunk
 
+                    # yield once for an empty file
+                    if is_empty:
+                        yield str(dest), b""
+
     def _stream_folder_from_url(self, chunk_size=8192):
         if not self.fetch_remote:
             if self.validate_url:
@@ -124,8 +130,14 @@ def _stream_folder_from_url(self, chunk_size=8192):
                     part_uri = f"{base}/{part}"
                     rel_out_path = Path(self.id) / part
 
+                    is_empty = True
                     with urlopen(part_uri) as response:
                         while chunk := response.read(chunk_size):
+                            is_empty = False
                             yield str(rel_out_path), chunk
+
+                    # yield once for an empty file
+                    if is_empty:
+                        yield str(rel_out_path), b""
                 except KeyError:
                     warnings.warn(f"'hasPart' entry in {self.id} is missing '@id'. Skipping.")

rocrate/model/file.py

@@ -116,6 +116,10 @@ def _stream_from_url(self, url, chunk_size=8192):
                         yield self.id, chunk
                         size += len(chunk)
 
+                # yield once for an empty file
+                if size == 0:
+                    yield self.id, b""
+
                 if self.record_size:
                     self._jsonld['contentSize'] = str(size)
 
@@ -126,6 +130,11 @@ def _stream_from_file(self, path, chunk_size=8192):
             while chunk := f.read(chunk_size):
                 yield self.id, chunk
                 size += len(chunk)
+
+        # yield once for an empty file
+        if size == 0:
+            yield self.id, b""
+
         if self.record_size:
             self._jsonld['contentSize'] = str(size)
 

test/test_write.py

@@ -525,6 +525,7 @@ def test_percent_escape(test_data_dir, tmpdir, helpers):
     assert (out_path / "a b" / "c d.txt").is_file()
     assert (out_path / "subdir" / "a b" / "c d.txt").is_file()
 
+
 def test_stream_empty_file(test_data_dir, tmpdir):
     """
     Test that empty files are written correctly to the zip file.
@@ -534,7 +535,7 @@ def test_stream_empty_file(test_data_dir, tmpdir):
     crate.add_file(crate_dir / "empty.txt")
     crate.add_directory(crate_dir / "folder")
 
-    # write the crate to a zip file 
+    # write the crate to a zip file
     out_path = tmpdir / 'ro_crate_out.zip'
     crate.write_zip(out_path)
 
@@ -546,4 +547,4 @@ def test_stream_empty_file(test_data_dir, tmpdir):
             files_in_zip[info.filename] = info.file_size
 
     assert files_in_zip["empty.txt"] == 0
-    assert files_in_zip["folder/empty_not_listed.txt"] == 0
+    assert files_in_zip["folder/empty_not_listed.txt"] == 0