Remove archive after it is extracted to save disk space (#1351)

* Remove archive after it is extracted to save disk space

* Leave a marker after removing archive to avoid redownload

* Automatic refresh if expected marker is absent

* Be consistent about syntax use for path construction

Author: PGijsbers

openml/_api_calls.py

@@ -6,6 +6,7 @@
 import logging
 import math
 import random
+import shutil
 import time
 import urllib.parse
 import xml
@@ -186,14 +187,14 @@ def _download_minio_file(
 def _download_minio_bucket(source: str, destination: str | Path) -> None:
     """Download file ``source`` from a MinIO Bucket and store it at ``destination``.
 
+    Does not redownload files which already exist.
+
     Parameters
     ----------
     source : str
         URL to a MinIO bucket.
     destination : str | Path
         Path to a directory to store the bucket content in.
-    exists_ok : bool, optional (default=True)
-        If False, raise FileExists if a file already exists in ``destination``.
     """
     destination = Path(destination)
     parsed_url = urllib.parse.urlparse(source)
@@ -206,15 +207,28 @@ def _download_minio_bucket(source: str, destination: str | Path) -> None:
 
     for file_object in client.list_objects(bucket, prefix=prefix, recursive=True):
         if file_object.object_name is None:
-            raise ValueError("Object name is None.")
+            raise ValueError(f"Object name is None for object {file_object!r}")
 
-        with contextlib.suppress(FileExistsError):  # Simply use cached version instead
+        marker = destination / file_object.etag
+        if marker.exists():
+            continue
+
+        file_destination = destination / file_object.object_name.rsplit("/", 1)[1]
+        if (file_destination.parent / file_destination.stem).exists():
+            # Marker is missing but archive exists means the server archive changed, force a refresh
+            shutil.rmtree(file_destination.parent / file_destination.stem)
+
+        with contextlib.suppress(FileExistsError):
             _download_minio_file(
                 source=source.rsplit("/", 1)[0] + "/" + file_object.object_name.rsplit("/", 1)[1],
-                destination=Path(destination, file_object.object_name.rsplit("/", 1)[1]),
+                destination=file_destination,
                 exists_ok=False,
             )
 
+        if file_destination.is_file() and file_destination.suffix == ".zip":
+            file_destination.unlink()
+            marker.touch()
+
 
 def _download_text_file(
     source: str,