running pipenv run black

Author: tcnichol

scripts/migration/dataset_collection_json.py

@@ -9,7 +9,7 @@
 except ImportError:
     import tomli as tomllib
 
-path_to_env = os.path.join(os.getcwd(),"scripts","migration", ".env")
+path_to_env = os.path.join(os.getcwd(), "scripts", "migration", ".env")
 config = dotenv_values(dotenv_path=path_to_env)
 
 CLOWDER_V1 = config["CLOWDER_V1"]
@@ -32,23 +32,26 @@
 timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
 COLLECTIONS_FILE = "collections_datasets.json"
 
+
 def get_all_datasets(header):
     endpoint = f"{CLOWDER_V1}/api/datasets?superAdmin=true&limit=0"
     datasets = requests.get(endpoint, headers=header).json()
     return datasets
 
+
 def get_dataset_metadata(dataset_v1_id, headers_v1):
     # Get metadata from Clowder V1
     endpoint = (
         f"{CLOWDER_V1}/api/datasets/{dataset_v1_id}/metadata.jsonld?superAdmin=true"
     )
     metadata_v1 = requests.get(endpoint, headers=headers_v1).json()
     if len(metadata_v1) > 0:
-        print('we got some metadata')
-        with open('datasets_with_metadata.txt', 'a') as f:
-            f.write(dataset_v1_id + '\n')
+        print("we got some metadata")
+        with open("datasets_with_metadata.txt", "a") as f:
+            f.write(dataset_v1_id + "\n")
     return metadata_v1
 
+
 def get_dataset_collections_map():
     print("Getting collections and datasets from Clowder v1...")
 
@@ -67,19 +70,22 @@ def get_dataset_collections_map():
                 dataset_to_collection[dataset] = current_value
     return dataset_to_collection
 
+
 def get_datasets_in_collections():
     map = get_dataset_collections_map()
     datasets_in_collections = list(map.keys())
     datasets_with_metadata = []
     for i in range(0, len(datasets_in_collections)):
         current_dataset = datasets_in_collections[i]
-        dataset_metadata = get_dataset_metadata(current_dataset, base_headers_v1, datasets_with_metadata)
+        dataset_metadata = get_dataset_metadata(
+            current_dataset, base_headers_v1, datasets_with_metadata
+        )
     return datasets_in_collections
 
+
 if __name__ == "__main__":
     all_datasets = get_all_datasets(base_headers_v1)
     for i in range(0, len(all_datasets)):
         current_dataset = all_datasets[i]
-        get_dataset_metadata(current_dataset['id'], base_headers_v1)
+        get_dataset_metadata(current_dataset["id"], base_headers_v1)
     get_datasets_in_collections()
-

scripts/migration/get_collections.py

@@ -24,7 +24,7 @@
 OUTPUT_FILE = "collections_ids.txt"
 
 # Load environment variables
-path_to_env = os.path.join(os.getcwd(),"scripts","migration", ".env")
+path_to_env = os.path.join(os.getcwd(), "scripts", "migration", ".env")
 config = dotenv_values(dotenv_path=path_to_env)
 
 
@@ -60,12 +60,14 @@
     "last_name": "admin",
 }
 
+
 def get_clowder_v1_top_level_collections(headers):
     endpoint = f"{CLOWDER_V1}/api/collections/topLevelCollections?superAdmin=true"
     response = requests.get(endpoint, headers=headers)
     user_collections = response.json()
     return user_collections
 
+
 def get_collection_v1_descendants(headers, collection_id):
     descendant_ids = []
 
@@ -75,17 +77,20 @@ def get_collection_v1_descendants(headers, collection_id):
     print(collection_json["child_collection_ids"])
     if int(collection_json["childCollectionsCount"]) > 0:
         child_collections_ids = collection_json["child_collection_ids"]
-        descendant_ids = child_collections_ids[5:-1].split(', ')
+        descendant_ids = child_collections_ids[5:-1].split(", ")
         for i in range(0, len(descendant_ids)):
             id = descendant_ids[i]
             descendent_endpoint = f"{CLOWDER_V1}/api/collections/{id}"
-            descendent_response = requests.get(descendent_endpoint, headers=headers, verify=False)
+            descendent_response = requests.get(
+                descendent_endpoint, headers=headers, verify=False
+            )
             descendent_json = descendent_response.json()
             if int(descendent_json["childCollectionsCount"]) > 0:
                 sub_descendants = get_collection_v1_descendants(headers, id)
                 descendant_ids.extend(sub_descendants)
     return descendant_ids
 
+
 def get_dataset_ids_in_v1_collection(headers, collection_id):
     dataset_ids = []
     collection_endpoint = f"{CLOWDER_V1}/api/collections/{collection_id}/datasets"
@@ -95,17 +100,23 @@ def get_dataset_ids_in_v1_collection(headers, collection_id):
         dataset_ids.append(dataset["id"])
     return dataset_ids
 
+
 if __name__ == "__main__":
     top_level_collections = get_clowder_v1_top_level_collections(clowder_headers_v1)
     all_v1_collections = []
     for collection in top_level_collections:
-        print(f"Getting descendents for collection {collection['name']} ({collection['id']})")
+        print(
+            f"Getting descendents for collection {collection['name']} ({collection['id']})"
+        )
         all_v1_collections.append(collection["id"])
         if int(collection["childCollectionsCount"]) > 0:
-            descendant_ids = get_collection_v1_descendants(clowder_headers_v1, collection["id"])
+            descendant_ids = get_collection_v1_descendants(
+                clowder_headers_v1, collection["id"]
+            )
             all_v1_collections.extend(descendant_ids)
-            print(f"Added descendents for collection {collection['name']} ({collection['id']})")
-
+            print(
+                f"Added descendents for collection {collection['name']} ({collection['id']})"
+            )
 
     print(f"TOTAL V1 COLLECTIONS TO MIGRATE: {len(all_v1_collections)}")
 

scripts/migration/get_collections_datasets.py

@@ -10,17 +10,15 @@
     import tomli as tomllib
 
 
-
 DEFAULT_PASSWORD = "Password123&"
 
 # Get the current timestamp
 timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
 COLLECTIONS_FILE = "collections_ids.txt"
 
 
-
 # Load environment variables
-path_to_env = os.path.join(os.getcwd(),"scripts","migration", ".env")
+path_to_env = os.path.join(os.getcwd(), "scripts", "migration", ".env")
 config = dotenv_values(dotenv_path=path_to_env)
 
 
@@ -56,6 +54,7 @@
     "last_name": "admin",
 }
 
+
 def get_collections_datasets(headers, collection_id):
     collection_dataset_endpoint = (
         f"{CLOWDER_V1}/api/collections/{collection_id}/datasets?superAdmin=true"
@@ -70,16 +69,16 @@ def get_collections_datasets(headers, collection_id):
 if __name__ == "__main__":
     print("Getting collections and datasets from Clowder v1...")
 
-    collection_ids =[]
+    collection_ids = []
     if os.path.exists(COLLECTIONS_FILE):
-        print('exists')
+        print("exists")
     else:
-        print('does not exist')
+        print("does not exist")
 
     with open(COLLECTIONS_FILE, "r") as outfile:
         lines = outfile.readlines()
         for line in lines:
-            collection_ids.append(line.rstrip('\n'))
+            collection_ids.append(line.rstrip("\n"))
     print(f"Found {len(collection_ids)} collections in {COLLECTIONS_FILE}")
     collection_dataset_dict = dict()
     for id in collection_ids:
@@ -94,4 +93,4 @@ def get_collections_datasets(headers, collection_id):
     json_file = "collections_datasets.json"
     with open(json_file, "w") as jf:
         json.dump(collection_dataset_dict, jf)
-    print("dumped to a file")
+    print("dumped to a file")