Skip to content

Commit 851503e

Browse files
tcnicholtcnichol
committed
refactoring migrate_user
this might need to be modified for CILogon, but this will work for local users
1 parent e216405 commit 851503e

File tree

1 file changed

+114
-94
lines changed

1 file changed

+114
-94
lines changed

‎scripts/migration/migrate_users.py‎

Lines changed: 114 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -327,119 +327,139 @@ async def add_dataset_folders(dataset_v1, dataset_v2, current_headers):
327327
print("added", folder)
328328

329329

330-
async def process_users(
330+
async def process_user(
331+
user_v1,
331332
fs: Minio = Depends(dependencies.get_fs),
332333
es: Elasticsearch = Depends(dependencies.get_elasticsearchclient),
333334
rabbitmq_client: BlockingChannel = Depends(dependencies.get_rabbitmq),
334335
):
335-
print("We create a v2 admin user")
336-
NEW_ADMIN_KEY_V2 = create_admin_user()
337-
users_v1 = get_clowder_v1_users()
338-
for user_v1 in users_v1:
339-
print("migrating v1 user", user_v1)
340-
id = user_v1["id"]
341-
email = user_v1["email"]
342-
firstName = user_v1["firstName"]
343-
lastName = user_v1["lastName"]
344-
345-
id_provider = user_v1["identityProvider"]
346-
if "[Local Account]" in user_v1["identityProvider"]:
347-
# get the v2 users
348-
# create a user account in v2 with this username
349-
if email != "[email protected]":
350-
user_v1_datasets = get_clowder_v1_user_datasets(user_id=id)
351-
# TODO check if there is already a local user
352-
user_v2 = create_local_user(user_v1)
353-
user_v2_api_key = user_v2
354-
user_base_headers_v2 = {"X-API-key": user_v2_api_key}
355-
user_headers_v2 = {
356-
**user_base_headers_v2,
357-
"Content-type": "application/json",
358-
"accept": "application/json",
359-
}
360-
for dataset in user_v1_datasets:
361-
print("creating a dataset in v2", dataset["id"], dataset["name"])
362-
dataset_v2_id = await create_v2_dataset(
363-
user_base_headers_v2, dataset, email
364-
)
365-
folders = await add_dataset_folders(
366-
dataset, dataset_v2_id, user_headers_v2
367-
)
368-
print("Created folders in new dataset")
369-
370-
all_dataset_folders = await get_folder_and_subfolders(
371-
dataset_id=dataset_v2_id,
372-
folder=None,
373-
current_headers=user_headers_v2,
374-
)
375-
dataset_files_endpoint = (
336+
print("migrating v1 user", user_v1)
337+
id = user_v1["id"]
338+
email = user_v1["email"]
339+
firstName = user_v1["firstName"]
340+
lastName = user_v1["lastName"]
341+
342+
id_provider = user_v1["identityProvider"]
343+
if "[Local Account]" in user_v1["identityProvider"]:
344+
# get the v2 users
345+
# create a user account in v2 with this username
346+
if email != "[email protected]":
347+
user_v1_datasets = get_clowder_v1_user_datasets(user_id=id)
348+
# TODO check if there is already a local user
349+
user_v2 = create_local_user(user_v1)
350+
user_v2_api_key = user_v2
351+
user_base_headers_v2 = {"X-API-key": user_v2_api_key}
352+
user_headers_v2 = {
353+
**user_base_headers_v2,
354+
"Content-type": "application/json",
355+
"accept": "application/json",
356+
}
357+
for dataset in user_v1_datasets:
358+
print("creating a dataset in v2", dataset["id"], dataset["name"])
359+
dataset_v2_id = await create_v2_dataset(
360+
user_base_headers_v2, dataset, email
361+
)
362+
folders = await add_dataset_folders(
363+
dataset, dataset_v2_id, user_headers_v2
364+
)
365+
print("Created folders in new dataset")
366+
367+
all_dataset_folders = await get_folder_and_subfolders(
368+
dataset_id=dataset_v2_id,
369+
folder=None,
370+
current_headers=user_headers_v2,
371+
)
372+
dataset_files_endpoint = (
376373
CLOWDER_V1
377374
+ "api/datasets/"
378375
+ dataset["id"]
379376
+ "/files?=superAdmin=true"
377+
)
378+
r_files = requests.get(
379+
dataset_files_endpoint, headers=clowder_headers_v1, verify=False
380+
)
381+
files_result = r_files.json()
382+
for file in files_result:
383+
file_folder = None
384+
file_id = file["id"]
385+
filename = file["filename"]
386+
if "folders" in file:
387+
file_folder = file["folders"]
388+
# TODO download the file from v1 using api routes
389+
v1_download_url = (
390+
CLOWDER_V1 + "api/files/" + file_id + "?superAdmin=true"
380391
)
381-
r_files = requests.get(
382-
dataset_files_endpoint, headers=clowder_headers_v1, verify=False
392+
print("downloading file", filename)
393+
download = requests.get(
394+
v1_download_url, headers=clowder_headers_v1
383395
)
384-
files_result = r_files.json()
385-
for file in files_result:
386-
file_folder = None
387-
file_id = file["id"]
388-
filename = file["filename"]
389-
if "folders" in file:
390-
file_folder = file["folders"]
391-
# TODO download the file from v1 using api routes
392-
v1_download_url = (
393-
CLOWDER_V1 + "api/files/" + file_id + "?superAdmin=true"
394-
)
395-
print("downloading file", filename)
396-
download = requests.get(
397-
v1_download_url, headers=clowder_headers_v1
398-
)
399-
with open(filename, "wb") as f:
400-
f.write(download.content)
401-
file_data = {"file": open(filename, "rb")}
402-
matching_folder = None
403-
if file_folder:
404-
for folder in all_dataset_folders:
405-
if folder["name"] == file_folder["name"]:
406-
matching_folder = folder
407-
if matching_folder:
408-
upload_files = {"files": open(filename, "rb")}
409-
dataset_file_upload_endpoint = (
396+
with open(filename, "wb") as f:
397+
f.write(download.content)
398+
file_data = {"file": open(filename, "rb")}
399+
matching_folder = None
400+
if file_folder:
401+
for folder in all_dataset_folders:
402+
if folder["name"] == file_folder["name"]:
403+
matching_folder = folder
404+
if matching_folder:
405+
upload_files = {"files": open(filename, "rb")}
406+
dataset_file_upload_endpoint = (
410407
CLOWDER_V2
411408
+ "api/v2/datasets/"
412409
+ dataset_v2_id
413410
+ "/filesMultiple?folder_id="
414411
+ matching_folder["id"]
415-
)
416-
response = requests.post(
417-
dataset_file_upload_endpoint,
418-
files=upload_files,
419-
headers=user_base_headers_v2,
420-
)
421-
422-
else:
423-
dataset_file_upload_endpoint = (
412+
)
413+
response = requests.post(
414+
dataset_file_upload_endpoint,
415+
files=upload_files,
416+
headers=user_base_headers_v2,
417+
)
418+
419+
else:
420+
dataset_file_upload_endpoint = (
424421
CLOWDER_V2
425422
+ "api/v2/datasets/"
426423
+ dataset_v2_id
427424
+ "/files"
428-
)
429-
response = requests.post(
430-
dataset_file_upload_endpoint,
431-
files=file_data,
432-
headers=user_base_headers_v2,
433-
)
434-
if response.status_code == 200:
435-
result = response.json()
436-
print("added file", result)
437-
try:
438-
os.remove(filename)
439-
except Exception as e:
440-
print("could not delete locally downloaded file")
441-
print(e)
442-
print("done with file upload")
425+
)
426+
response = requests.post(
427+
dataset_file_upload_endpoint,
428+
files=file_data,
429+
headers=user_base_headers_v2,
430+
)
431+
if response.status_code == 200:
432+
result = response.json()
433+
print("added file", result)
434+
try:
435+
os.remove(filename)
436+
except Exception as e:
437+
print("could not delete locally downloaded file")
438+
print(e)
439+
print("done with file upload")
440+
441+
442+
async def process_users(
443+
fs: Minio = Depends(dependencies.get_fs),
444+
es: Elasticsearch = Depends(dependencies.get_elasticsearchclient),
445+
rabbitmq_client: BlockingChannel = Depends(dependencies.get_rabbitmq),
446+
):
447+
print("We create a v2 admin user")
448+
NEW_ADMIN_KEY_V2 = create_admin_user()
449+
users_v1 = get_clowder_v1_users()
450+
for user_v1 in users_v1:
451+
print("migrating v1 user", user_v1)
452+
id = user_v1["id"]
453+
email = user_v1["email"]
454+
firstName = user_v1["firstName"]
455+
lastName = user_v1["lastName"]
456+
457+
id_provider = user_v1["identityProvider"]
458+
if "[Local Account]" in user_v1["identityProvider"]:
459+
# get the v2 users
460+
# create a user account in v2 with this username
461+
await process_user(user_v1, fs=fs, es=es, rabbitmq_client=rabbitmq_client)
462+
print("Migrated user", user_v1)
443463

444464
else:
445465
print("not a local account, not migrated at this time")

0 commit comments

Comments
 (0)