Merge pull request #227 from simleo/fix_unescaping

Fix percent unescaping

Author: simleo

rocrate/model/dataset.py

@@ -69,7 +69,10 @@ def _copy_folder(self, base_path):
         if self.source is None:
             abs_out_path.mkdir(parents=True, exist_ok=True)
         else:
-            path = unquote(str(self.source))
+            if self.crate.mode == Mode.READ:
+                path = unquote(str(self.source))
+            else:
+                path = self.source
             if not Path(path).exists():
                 raise FileNotFoundError(
                     errno.ENOENT, os.strerror(errno.ENOENT), path
@@ -94,7 +97,10 @@ def stream(self, chunk_size=8192):
             yield from self._stream_folder_from_path(chunk_size)
 
     def _stream_folder_from_path(self, chunk_size=8192):
-        path = unquote(str(self.source))
+        if self.crate.mode == Mode.READ:
+            path = unquote(str(self.source))
+        else:
+            path = self.source
         if not Path(path).exists():
             raise FileNotFoundError(
                 errno.ENOENT, os.strerror(errno.ENOENT), str(path)
@@ -104,7 +110,7 @@ def _stream_folder_from_path(self, chunk_size=8192):
                 root = Path(root)
                 for name in files:
                     source = root / name
-                    dest = source.relative_to(Path(path).parent)
+                    dest = Path(unquote(self.id)) / source.relative_to(path)
                     is_empty = True
                     with open(source, 'rb') as f:
                         while chunk := f.read(chunk_size):

rocrate/model/file.py

@@ -31,7 +31,7 @@
 from urllib.parse import unquote
 
 from .file_or_dir import FileOrDir
-from ..utils import is_url, iso_now
+from ..utils import is_url, iso_now, Mode
 
 
 class File(FileOrDir):
@@ -64,7 +64,6 @@ def _write_from_stream(self, out_file_path):
                 out_file.write(chunk)
 
     def _copy_file(self, path, out_file_path):
-        path = unquote(str(path))
         out_file_path.parent.mkdir(parents=True, exist_ok=True)
         if not out_file_path.exists() or not out_file_path.samefile(path):
             shutil.copy(path, out_file_path)
@@ -79,7 +78,11 @@ def write(self, base_path):
             # Allows to record a File entity whose @id does not exist, see #73
             warnings.warn(f"No source for {self.id}")
         else:
-            self._copy_file(self.source, out_file_path)
+            if self.crate.mode == Mode.READ:
+                in_file_path = unquote(str(self.source))
+            else:
+                in_file_path = self.source
+            self._copy_file(in_file_path, out_file_path)
 
     def _stream_from_stream(self, stream):
         size = 0
@@ -124,11 +127,10 @@ def _stream_from_url(self, url, chunk_size=8192):
                     self._jsonld['contentSize'] = str(size)
 
     def _stream_from_file(self, path, chunk_size=8192):
-        path = unquote(str(path))
         size = 0
         with open(path, 'rb') as f:
             while chunk := f.read(chunk_size):
-                yield self.id, chunk
+                yield unquote(self.id), chunk
                 size += len(chunk)
 
         # yield once for an empty file
@@ -147,4 +149,8 @@ def stream(self, chunk_size=8192):
             # Allows to record a File entity whose @id does not exist, see #73
             warnings.warn(f"No source for {self.id}")
         else:
-            yield from self._stream_from_file(self.source, chunk_size)
+            if self.crate.mode == Mode.READ:
+                path = unquote(str(self.source))
+            else:
+                path = self.source
+            yield from self._stream_from_file(path, chunk_size)

test/test-data/read_crate/j%20k/l%20m.txt

@@ -0,0 +1 @@
+L%20M

test/test-data/read_crate/without%20space.txt

@@ -0,0 +1 @@
+The name of this file should NOT be unescaped

test/test_write.py

@@ -510,20 +510,48 @@ def test_percent_escape(test_data_dir, tmpdir, helpers):
     assert f1.id == "with%20space.txt"
     f2 = crate.add_file(f_path, dest_path="subdir/with space.txt")
     assert f2.id == "subdir/with%20space.txt"
+    f3 = crate.add_file(test_data_dir / "read_crate" / "without%20space.txt")
+    assert f3.id == "without%2520space.txt"
     d_path = test_data_dir / "read_crate" / "a b"
     d1 = crate.add_dataset(d_path)
     assert d1.id == "a%20b/"
     d2 = crate.add_dataset(d_path, dest_path="subdir/a b")
     assert d2.id == "subdir/a%20b/"
+    d3 = crate.add_dataset(test_data_dir / "read_crate" / "j%20k")
+    assert d3.id == "j%2520k/"
     out_path = tmpdir / "ro_crate_out"
     crate.write(out_path)
     json_entities = helpers.read_json_entities(out_path)
     assert "with%20space.txt" in json_entities
     assert "subdir/with%20space.txt" in json_entities
+    assert "without%2520space.txt" in json_entities
     assert "a%20b/" in json_entities
     assert "subdir/a%20b/" in json_entities
+    assert "j%2520k/" in json_entities
+    assert (out_path / "with space.txt").is_file()
+    assert (out_path / "subdir" / "with space.txt").is_file()
+    assert (out_path / "without%20space.txt").is_file()
     assert (out_path / "a b" / "c d.txt").is_file()
     assert (out_path / "subdir" / "a b" / "c d.txt").is_file()
+    assert (out_path / "j%20k" / "l%20m.txt").is_file()
+    out_zip_path = tmpdir / "ro_crate_out.zip"
+    crate.write_zip(out_zip_path)
+    unpack_path = tmpdir / "unpack"
+    with zipfile.ZipFile(out_zip_path, "r") as zf:
+        zf.extractall(unpack_path)
+    json_entities = helpers.read_json_entities(unpack_path)
+    assert "with%20space.txt" in json_entities
+    assert "subdir/with%20space.txt" in json_entities
+    assert "without%2520space.txt" in json_entities
+    assert "a%20b/" in json_entities
+    assert "subdir/a%20b/" in json_entities
+    assert "j%2520k/" in json_entities
+    assert (unpack_path / "with space.txt").is_file()
+    assert (unpack_path / "subdir" / "with space.txt").is_file()
+    assert (unpack_path / "without%20space.txt").is_file()
+    assert (unpack_path / "a b" / "c d.txt").is_file()
+    assert (unpack_path / "subdir" / "a b" / "c d.txt").is_file()
+    assert (unpack_path / "j%20k" / "l%20m.txt").is_file()
 
 
 def test_stream_empty_file(test_data_dir, tmpdir):
@@ -548,3 +576,24 @@ def test_stream_empty_file(test_data_dir, tmpdir):
 
     assert files_in_zip["empty.txt"] == 0
     assert files_in_zip["folder/empty_not_listed.txt"] == 0
+
+
+def test_write_zip_nested_dest(tmpdir, helpers):
+    root = tmpdir / "root"
+    root.mkdir()
+    (root / "a b").mkdir()
+    (root / "a b" / "c d.txt").write_text("C D\n")
+    (root / "a b" / "j k").mkdir()
+    (root / "a b" / "j k" / "l m.txt").write_text("L M\n")
+    crate = ROCrate()
+    d1 = crate.add_dataset(root / "a b", dest_path="subdir/a b")
+    assert d1.id == "subdir/a%20b/"
+    out_zip_path = tmpdir / "ro_crate_out.zip"
+    crate.write_zip(out_zip_path)
+    unpack_path = tmpdir / "unpack"
+    with zipfile.ZipFile(out_zip_path, "r") as zf:
+        zf.extractall(unpack_path)
+    json_entities = helpers.read_json_entities(unpack_path)
+    assert "subdir/a%20b/" in json_entities
+    assert (unpack_path / "subdir" / "a b" / "c d.txt").is_file()
+    assert (unpack_path / "subdir" / "a b" / "j k" / "l m.txt").is_file()