migrate now adds a metadata file to a folder

Author: tcnichol

scripts/migration/dataset_collection_json.py

@@ -9,14 +9,46 @@
 except ImportError:
     import tomli as tomllib
 
+path_to_env = os.path.join(os.getcwd(),"scripts","migration", ".env")
+config = dotenv_values(dotenv_path=path_to_env)
 
+CLOWDER_V1 = config["CLOWDER_V1"]
+ADMIN_KEY_V1 = config["ADMIN_KEY_V1"]
+CLOWDER_V2 = config["CLOWDER_V2"]
+ADMIN_KEY_V2 = config["ADMIN_KEY_V2"]
+
+base_headers_v1 = {"X-API-key": ADMIN_KEY_V1}
+base_headers_v2 = {"X-API-key": ADMIN_KEY_V2}
+
+clowder_headers_v1 = {
+    **base_headers_v1,
+    "Content-type": "application/json",
+    "accept": "application/json",
+}
 
 DEFAULT_PASSWORD = "Password123&"
 
 # Get the current timestamp
 timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
 COLLECTIONS_FILE = "collections_datasets.json"
 
+def get_all_datasets(header):
+    endpoint = f"{CLOWDER_V1}/api/datasets?superAdmin=true&limit=0"
+    datasets = requests.get(endpoint, headers=header).json()
+    return datasets
+
+def get_dataset_metadata(dataset_v1_id, headers_v1):
+    # Get metadata from Clowder V1
+    endpoint = (
+        f"{CLOWDER_V1}/api/datasets/{dataset_v1_id}/metadata.jsonld?superAdmin=true"
+    )
+    metadata_v1 = requests.get(endpoint, headers=headers_v1).json()
+    if len(metadata_v1) > 0:
+        print('we got some metadata')
+        with open('datasets_with_metadata.txt', 'a') as f:
+            f.write(dataset_v1_id + '\n')
+    return metadata_v1
+
 def get_dataset_collections_map():
     print("Getting collections and datasets from Clowder v1...")
 
@@ -38,5 +70,16 @@ def get_dataset_collections_map():
 def get_datasets_in_collections():
     map = get_dataset_collections_map()
     datasets_in_collections = list(map.keys())
+    datasets_with_metadata = []
+    for i in range(0, len(datasets_in_collections)):
+        current_dataset = datasets_in_collections[i]
+        dataset_metadata = get_dataset_metadata(current_dataset, base_headers_v1, datasets_with_metadata)
     return datasets_in_collections
 
+if __name__ == "__main__":
+    all_datasets = get_all_datasets(base_headers_v1)
+    for i in range(0, len(all_datasets)):
+        current_dataset = all_datasets[i]
+        get_dataset_metadata(current_dataset['id'], base_headers_v1)
+    get_datasets_in_collections()
+

scripts/migration/migrate.py

@@ -1,6 +1,6 @@
 import os
 from datetime import datetime
-
+import json
 import requests
 from dotenv import dotenv_values
 
@@ -217,9 +217,11 @@ def process_collection_descendants(collection, headers_v1, base_headers_v2, head
             new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_parent_type, v2_dataset_id, headers_v2)
             process_dataset_files_and_folders(dataset, headers_v1, base_headers_v2, 'folder', new_folder['id'], v2_dataset_id, new_folder)
             # TODO add dataset metadata to the folder
+            add_dataset_metadata_to_folder(dataset, v2_dataset_id,  new_folder['id'], headers_v1, base_headers_v2)
         else:
             new_folder = create_folder_if_not_exists_or_get(dataset["name"], v2_parent_id, v2_parent_type, v2_dataset_id, headers_v2)
             process_dataset_files_and_folders(dataset, headers_v1, base_headers_v2, 'folder', new_folder['id'], v2_dataset_id, new_folder)
+            add_dataset_metadata_to_folder(dataset, v2_dataset_id,  new_folder['id'], headers_v1, base_headers_v2)
             # TODO add dataset metadata to the folder
 
 
@@ -798,7 +800,6 @@ def add_file_metadata(file_v1, file_v2_id, headers_v1, headers_v2):
                         print("Successfully posted file machine metadata to V2")
                     break  # machine metadata no need to iterate through all the keys
 
-
 def add_dataset_metadata(dataset_v1, dataset_v2_id, headers_v1, headers_v2):
     # Get metadata from Clowder V1
     endpoint = (
@@ -853,6 +854,46 @@ def add_dataset_metadata(dataset_v1, dataset_v2_id, headers_v1, headers_v2):
                     break  # machine metadata no need to iterate through all the keys
 
 
+def add_dataset_metadata_to_folder(dataset_v1, dataset_v2_id, folder_v2_id, headers_v1, headers_v2):
+    # Get metadata from Clowder V1
+    endpoint = (
+        f"{CLOWDER_V1}/api/datasets/{dataset_v1['id']}/metadata.jsonld?superAdmin=true"
+    )
+    dataset_name = dataset_v1['name']
+    metadata_file_name = dataset_name + '_metadata.json'
+    metadata_v1 = requests.get(endpoint, headers=headers_v1).json()
+    with open(metadata_file_name, "w") as metadata_file:
+        json.dump(metadata_v1, metadata_file)
+
+    # upload the file to the folder in v2
+    dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/filesMultiple?folder_id={folder_v2_id}"
+
+    response = requests.post(
+        dataset_file_upload_endpoint,
+        headers=headers_v2,
+        files=[("files", (metadata_file_name, open(metadata_file_name, "rb")))],
+    )
+
+    # Clean up the local file after upload
+    print(f"Type response {type(response)}")
+    try:
+        os.remove(metadata_file_name)
+    except Exception as e:
+        print(f"Could not delete locally created metadata file: {metadata_file_name}")
+        print(e)
+
+    if response.status_code == 200:
+        print(f"Uploaded file: {metadata_file_name} to dataset {dataset_v2_id} and folder {folder_v2_id}")
+        response_json = response.json()
+        if type(response_json) == dict:
+            return response.json().get("id")
+        elif type(response_json) == list:
+            return response_json[0].get("id")
+    else:
+        print(f"Failed to upload file: {metadata_file} to dataset {dataset_v2_id} and folder {folder_v2_id}")
+    return None
+
+
 def register_migration_extractor():
     """Register the migration extractor in Clowder v2."""
     migration_extractor = {