Add script for data upload

Author: constantinpape

scripts/ome_challenge/.gitignore

@@ -1 +1,2 @@
 credentials.json
+export_minio.sh

scripts/ome_challenge/convert_data.py

@@ -51,8 +51,10 @@ def convert_to_ome_zarr_v3(name):
 
 
 def main():
-    name = "Platynereis-H2B-TL"
-    # convert_to_ome_zarr_v2(name)
+    # name = "Platynereis-H2B-TL"
+    # name = "Zebrafish-XSPIM-multiview"
+    name = "Zebrafish-H2B-short-timelapse"
+    convert_to_ome_zarr_v2(name)
     convert_to_ome_zarr_v3(name)
 
 

scripts/ome_challenge/download_data.py

@@ -6,7 +6,10 @@
 
 IDS = {
     "Platynereis-H2B-TL": "1jGwaJ62w80GYo5I_Jcb3O_g7y4RKEhjI",
-    "Zebrafish-XSPIM-multiview": "https://drive.google.com/drive/folders/175hZRrUNWM2UzY0wzXPFjuFZ5QKUN-tm?usp=drive_link"  # noqa
+    "Zebrafish-XSPIM-multiview": "175hZRrUNWM2UzY0wzXPFjuFZ5QKUN-tm",  # noqa
+    "Zebrafish-H2B-short-timelapse": "18fGJwQ0i5pBHQO8FHUuFcxqWeD7uwiBM",  # noqa
+    # This doesn't work.
+    # "Zebrafish-H2B-short-4views": "1iyMMCZO1rmamVGNVThJWElJKSKXROscF"
 }
 
 # ROOT = "/mnt/lustre-grete/usr/u12086/data/flamingo"
@@ -17,6 +20,7 @@ def download_folder(drive, name):
     os.makedirs(ROOT, exist_ok=True)
 
     destination_folder = os.path.join(ROOT, name)
+    os.makedirs(destination_folder, exist_ok=True)
     folder_id = IDS[name]
 
     folder_query = f"'{folder_id}' in parents and trashed=false"
@@ -48,7 +52,9 @@ def main():
     drive = get_drive()
 
     # download_from_gdrive(name="Zebrafish-XSPIM-multiview")
-    download_folder(drive, name="Platynereis-H2B-TL")
+    # download_folder(drive, name="Platynereis-H2B-TL")
+    # download_folder(drive, name="Zebrafish-H2B-short-timelapse")
+    download_folder(drive, name="Zebrafish-H2B-short-4views")
 
 
 if __name__ == "__main__":

scripts/ome_challenge/upload_data.py

@@ -0,0 +1,33 @@
+import os
+from subprocess import run
+
+ROOT = "/mnt/lustre-emmy-hdd/usr/u12086/data/flamingo"
+
+
+def upload_data(name):
+    data_root = os.path.join(ROOT, "ngff-v3", name)
+    assert os.path.exists(data_root), data_root
+
+    bucket_name = "n4bi-goe"
+
+    # Create the bucket.
+    cmd = [
+        "mc-client", "mb", f"challenge/{bucket_name}/{name}/"
+    ]
+    run(cmd)
+
+    # Run the copy.
+    cmd = [
+        "mc-client", "cp", "--recursive",
+        f"{data_root}/", f"challenge/{bucket_name}/{name}/"
+    ]
+    run(cmd)
+
+
+def main():
+    name = "Platynereis-H2B-TL.ome.zarr"
+    upload_data(name)
+
+
+if __name__ == "__main__":
+    main()