removing unused methods

tcnichol · tcnichol · commit 7e139cbc4bb2 · 2025-09-03T14:40:02.000-05:00
diff --git a/scripts/migration/migrate.py b/scripts/migration/migrate.py
@@ -221,13 +221,6 @@ def process_collection_descendants(collection, headers_v1, base_headers_v2, head
             process_dataset_files_and_folders(dataset, headers_v1, base_headers_v2, 'folder', new_folder['id'], v2_dataset_id, new_folder)
 
 
-
-def process_dataset_folders(dataset, headers_v1, headers_v2, parent_type, parent_id):
-    folder_endpoint = f"{CLOWDER_V1}/api/datasets/{dataset['id']}/folders"
-    folder_response = requests.get(folder_endpoint, headers=headers_v1)
-    folder_json = folder_response.json()
-    print(f"Got dataset folders")
-
 def get_v1_dataset_folders(dataset, headers_v1, headers_v2, parent_type, parent_id):
     folder_endpoint = f"{CLOWDER_V1}/api/datasets/{dataset['id']}/folders"
     folder_response = requests.get(folder_endpoint, headers=headers_v1)
@@ -293,9 +286,6 @@ def create_v2_dataset_from_collection(collection, user_v1, headers_v1, headers_v
     return response.json()["id"]
 
 
-    # go through sub collections creating folders
-    print("Creating v2-dataset from collection")
-
 
 # TODO this is too slow, we need to optimize it
 def get_clowder_v1_dataset_collections(headers, user_v1, dataset_id):
@@ -544,7 +534,7 @@ def create_v2_group(space, headers):
     response = requests.post(group_in_v2_endpoint, json=group, headers=headers)
     return response.json()["id"]
 
-# TODO try this
+
 def add_folder_hierarchy_to_migration_folder(folder_hierarchy, dataset_v2, folder_id_v2, headers):
     """Add folder hierarchy to a dataset in Clowder v2."""
     hierarchy_parts = folder_hierarchy.split("/")
@@ -571,7 +561,7 @@ def add_folder_hierarchy(folder_hierarchy, dataset_v2, headers):
         if result:
             current_parent = result["id"]
 
-# for creating a folder for a dataset
+# for creating a folder for a dataset that is migrated to a dataset
 def create_dataset_folder_if_not_exists_or_get(folder, parent, dataset_v2, headers):
     """Create a folder if it does not exist or return the existing folder."""
     # current_folders = get_folder_and_subfolders(dataset_v2, headers)
@@ -591,6 +581,7 @@ def create_dataset_folder_if_not_exists_or_get(folder, parent, dataset_v2, heade
     )
     return response.json()
 
+# used for creating folders and subfolders when a collection is migrated to a dataset
 def create_folder_if_not_exists_or_get(folder, parent, parent_type, dataset_v2, headers):
     """Create a folder if it does not exist or return the existing folder."""
     # current_folders = get_folder_and_subfolders(dataset_v2, headers)
@@ -641,182 +632,6 @@ def add_dataset_folders(dataset_v1, dataset_v2, headers):
         add_folder_hierarchy(folder["name"], dataset_v2, headers)
 
 
-def download_and_upload_file_to_folder(file, folder, dataset_v2_id, headers_v2):
-    """Download a file from Clowder v1 and upload it to Clowder v2."""
-    filename = file["filename"]
-    file_id = file["id"]
-    file_folder = folder
-
-    # Download the file from Clowder v1
-    v1_download_url = f"{CLOWDER_V1}/api/files/{file_id}?superAdmin=true"
-    print(f"Downloading file: {filename}")
-    download_response = requests.get(v1_download_url, headers=clowder_headers_v1)
-
-    with open(filename, "wb") as f:
-        f.write(download_response.content)
-
-    # Upload the file to Clowder v2
-    dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/files"
-    if folder is not None:
-        # add folder if it is not None
-        folder_id = folder["id"]
-        dataset_file_upload_endpoint += f"Multiple?folder_id={folder_id}"
-    files = [
-        ("files", open(filename, "rb")),
-    ]
-    response = requests.post(
-        dataset_file_upload_endpoint,
-        headers=headers_v2,
-        files=files,
-    )
-    if response.status_code == 200:
-        print(f"Uploaded file: {filename} to dataset {dataset_v2_id}")
-        return response.json().get("id")
-    else:
-        print(f"Failed to upload file: {filename} to dataset {dataset_v2_id}")
-
-    # Clean up the local file after upload
-    try:
-        os.remove(filename)
-    except Exception as e:
-        print(f"Could not delete locally downloaded file: {filename}")
-        print(e)
-    return None
-
-def download_and_upload_file_to_folder_id(file, folder_v2, dataset_v2_id, headers_v2):
-    """Download a file from Clowder v1 and upload it to Clowder v2."""
-    filename = file["filename"]
-    file_id = file["id"]
-    file_folder = file.get("folders", None)
-
-    # Download the file from Clowder v1
-    v1_download_url = f"{CLOWDER_V1}/api/files/{file_id}?superAdmin=true"
-    print(f"Downloading file: {filename}")
-    download_response = requests.get(v1_download_url, headers=clowder_headers_v1)
-
-    with open(filename, "wb") as f:
-        f.write(download_response.content)
-
-    file_exists = os.path.exists(filename)
-    # Upload the file to Clowder v2
-    dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/files"
-    if folder_v2:
-        dataset_file_upload_endpoint += f"Multiple?folder_id={folder_v2['id']}"
-        response = requests.post(
-            dataset_file_upload_endpoint,
-            headers=headers_v2,
-            files=[("files", (filename, open(filename, "rb")))],
-        )
-    else:
-        response = requests.post(
-            dataset_file_upload_endpoint,
-            headers=headers_v2,
-            files={"file": open(filename, "rb")},
-        )
-
-    # Clean up the local file after upload
-    try:
-        os.remove(filename)
-    except Exception as e:
-        print(f"Could not delete locally downloaded file: {filename}")
-        print(e)
-
-    if response.status_code == 200:
-        print(f"Uploaded file: {filename} to dataset {dataset_v2_id}")
-        return response.json().get("id")
-    else:
-        print(f"Failed to upload file: {filename} to dataset {dataset_v2_id}")
-
-    return None
-
-# def download_and_upload_file_to_folder_id(file, folder_v2, dataset_v2_id, headers_v2):
-#     """Download a file from Clowder v1 and upload it to Clowder v2."""
-#     filename = file["filename"]
-#     file_id = file["id"]
-#
-#     # DEBUG: Print all inputs
-#     print(f"=== DEBUG START ===")
-#     print(f"File: {file}")
-#     print(f"Folder_v2: {folder_v2}")
-#     print(f"Dataset_v2_id: {dataset_v2_id}")
-#     print(f"Headers_v2 keys: {list(headers_v2.keys()) if headers_v2 else 'None'}")
-#
-#     # Download the file from Clowder v1
-#     v1_download_url = f"{CLOWDER_V1}/api/files/{file_id}?superAdmin=true"
-#     print(f"Downloading file: {filename} from {v1_download_url}")
-#     download_response = requests.get(v1_download_url, headers=clowder_headers_v1)
-#     print(f"Download status: {download_response.status_code}")
-#
-#     with open(filename, "wb") as f:
-#         f.write(download_response.content)
-#
-#     # Check file exists and has content
-#     file_size = os.path.getsize(filename)
-#     print(f"Local file size: {file_size} bytes")
-#
-#     # Upload the file to Clowder v2
-#     dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/files"
-#
-#     if folder_v2 is not None:
-#         folder_id = folder_v2['id'] if isinstance(folder_v2, dict) else folder_v2.id
-#         dataset_file_upload_endpoint += f"Multiple?folder_id={folder_id}"
-#
-#     print(f"Upload endpoint: {dataset_file_upload_endpoint}")
-#
-#     # Read file content to verify it's not corrupted
-#     with open(filename, "rb") as f:
-#         file_content = f.read()
-#         print(f"File content length: {len(file_content)}")
-#         print(f"File content starts with: {file_content[:100]}...")
-#
-#     # Make the upload request with detailed debugging
-#     with open(filename, "rb") as file_obj:
-#         files = {"file": (filename, file_obj)}
-#
-#         print(f"Final files dict: {files}")
-#         # Create headers without content-type for file uploads
-#         upload_headers = headers_v2.copy()
-#         upload_headers.pop('content-type', None)
-#         print(f"Final headers: {upload_headers}")
-#
-#         # Use a session to see raw request
-#         session = requests.Session()
-#         prepared_request = requests.Request(
-#             'POST',
-#             dataset_file_upload_endpoint,
-#             headers=upload_headers,
-#             files=files
-#         ).prepare()
-#
-#         print(f"Prepared request URL: {prepared_request.url}")
-#         print(f"Prepared request headers: {dict(prepared_request.headers)}")
-#         # Don't print body as it's binary, but we can check content-type
-#         print(f"Content-Type header: {prepared_request.headers.get('Content-Type')}")
-#
-#         response = session.send(prepared_request)
-#
-#     # DEBUG: Full response analysis
-#     print(f"Response status: {response.status_code}")
-#     print(f"Response headers: {dict(response.headers)}")
-#     print(f"Response text: {response.text}")
-#     print(f"=== DEBUG END ===")
-#
-#     # Clean up the local file after upload
-#     try:
-#         os.remove(filename)
-#     except Exception as e:
-#         print(f"Could not delete locally downloaded file: {filename}")
-#         print(e)
-#
-#     if response.status_code == 200:
-#         print(f"Uploaded file: {filename} to dataset {dataset_v2_id}")
-#         return response.json().get("id")
-#     else:
-#         print(f"Failed to upload file: {filename} to dataset {dataset_v2_id}")
-#
-#     return None
-
-
 def download_and_upload_file_to_matching_folder(file, dataset_v2_id, headers_v2, matching_folder = None):
     """Download a file from Clowder v1 and upload it to Clowder v2."""
     filename = file["filename"]
@@ -930,79 +745,6 @@ def download_and_upload_file(file, all_dataset_folders, dataset_v2_id, headers_v
 
     return None
 
-def download_and_upload_file_1(file, all_dataset_folders, dataset_v2_id, headers_v2):
-    """Download a file from Clowder v1 and upload it to Clowder v2."""
-    filename = file["filename"]
-    file_id = file["id"]
-    file_folder = file.get("folders", None)
-
-    # Download the file from Clowder v1
-    v1_download_url = f"{CLOWDER_V1}/api/files/{file_id}?superAdmin=true"
-    print(f"Downloading file: {filename}")
-    download_response = requests.get(v1_download_url, headers=clowder_headers_v1)
-
-    with open(filename, "wb") as f:
-        f.write(download_response.content)
-
-    # Determine the correct folder in Clowder v2 for the upload
-    matching_folder = None
-    if file_folder:
-        matching_folder = next(
-            (
-                folder
-                for folder in all_dataset_folders
-                if folder["name"] == file_folder["name"]
-            ),
-            None,
-        )
-
-    # Upload the file to Clowder v2
-    dataset_file_upload_endpoint = f"{CLOWDER_V2}/api/v2/datasets/{dataset_v2_id}/files"
-    if matching_folder:
-        dataset_file_upload_endpoint += f"Multiple?folder_id={matching_folder['id']}"
-
-    # DEBUG: Add the same debugging as the new method
-    print(f"=== WORKING METHOD DEBUG ===")
-    print(f"Upload endpoint: {dataset_file_upload_endpoint}")
-    print(f"Headers: {headers_v2}")
-
-    with open(filename, "rb") as file_obj:
-        files = {"file": (filename, file_obj)}
-
-        # Use a session to see raw request
-        session = requests.Session()
-        prepared_request = requests.Request(
-            'POST',
-            dataset_file_upload_endpoint,
-            headers=headers_v2,
-            files=files
-        ).prepare()
-
-        print(f"Prepared request URL: {prepared_request.url}")
-        print(f"Prepared request headers: {dict(prepared_request.headers)}")
-        print(f"Content-Type header: {prepared_request.headers.get('Content-Type')}")
-
-        response = session.send(prepared_request)
-
-    print(f"Response status: {response.status_code}")
-    print(f"Response text: {response.text}")
-    print(f"=== WORKING METHOD DEBUG END ===")
-
-    # Clean up the local file after upload
-    try:
-        os.remove(filename)
-    except Exception as e:
-        print(f"Could not delete locally downloaded file: {filename}")
-        print(e)
-
-    if response.status_code == 200:
-        print(f"Uploaded file: {filename} to dataset {dataset_v2_id}")
-        return response.json().get("id")
-    else:
-        print(f"Failed to upload file: {filename} to dataset {dataset_v2_id}")
-
-    return None
-
 def add_file_metadata(file_v1, file_v2_id, headers_v1, headers_v2):
     # Get metadata from Clowder V1
     endpoint = f"{CLOWDER_V1}/api/files/{file_v1['id']}/metadata.jsonld?superAdmin=true"
