excluding items

Author: tcnichol

scripts/migration/config.example.toml

@@ -1,13 +1,27 @@
 [spaces]
 space_ids = [
+    "66ec6210e4b08e5e86b30c20",
+    "62d992dce4b0e32fd16081f4"
+]
+exclude_space_ids = [
 
 ]
 
 [users]
-user_email = [
+user_emails = [
+    "[email protected]",
+    "[email protected]",
+    "[email protected]"
+
+]
+exclude_space_ids = [
+
 ]
 
-[exclude_users]
-exclude_user_emails = [
+[datasets]
+dataset_ids = [
 
-]
+]
+exclude_dataset_ids = [
+    "66eb1ddfe4b08e5e86b2e124"
+]

scripts/migration/migrate.py

@@ -25,8 +25,24 @@
 
 # Load environment variables
 path_to_env = os.path.join(os.getcwd(),"scripts","migration", ".env")
+path_to_toml = os.path.join(os.getcwd(),"scripts","migration", "config.toml")
 config = dotenv_values(dotenv_path=path_to_env)
 
+toml_space_ids=None
+toml_exclude_space_ids=None
+toml_users=None
+toml_exclude_users=None
+toml_exclude_dataset_ids=None
+
+# Load configuration from toml file
+if os.path.exists(path_to_toml):
+    toml_config = tomllib.loads(open(path_to_toml).read())
+    print(f"Loaded toml config")
+    toml_space_ids = toml_config["spaces"]["space_ids"]
+    toml_users = toml_config["users"]["user_emails"]
+    toml_exclude_dataset_ids = toml_config["datasets"]["exclude_dataset_ids"]
+
+
 CLOWDER_V1 = config["CLOWDER_V1"]
 ADMIN_KEY_V1 = config["ADMIN_KEY_V1"]
 CLOWDER_V2 = config["CLOWDER_V2"]
@@ -791,6 +807,14 @@ def process_user_and_resources(user_v1, USER_MAP, DATASET_MAP):
     USER_MAP = {}
     DATASET_MAP = {}
     users_v1 = get_clowder_v1_users()
+    # TODO filter if toml users
+    if toml_users is not None and len(toml_users) > 0:
+        print(f"Using spaces from config.toml: {toml_users}")
+        users_v1 = [
+            user for user in users_v1 if user["email"] in toml_users
+        ]
+    else:
+        print("No spaces specified in config.toml, migrating all users.")
     for user_v1 in users_v1:
         if (
             "[Local Account]" in user_v1["identityProvider"]