Fully reviewed and tested.

Author: RubelMozumder

run_uploader.sh

@@ -0,0 +1,12 @@
+#! /bin/bash
+
+# Stop on error
+set -e
+
+current_dir=$(pwd)
+uploader_script="/home/rubel/NOMAD-FAIRmat/nomad-distro-dev-RM/packages/pynxtools-spm/src/pynxtools_spm/nomad_uploader/example_upload_script.py"
+venv="/home/rubel/NOMAD-FAIRmat/nomad-distro-dev-RM/.venv"
+python_3="$venv/bin/python3"
+echo "Running uploader script..."
+"$python_3" "$uploader_script" > "$current_dir/debug.txt" 2>&1
+echo "Uploader script finished. Check debug.txt for details."

src/pynxtools_spm/nomad_uploader/example_upload_script.py

@@ -0,0 +1,51 @@
+from pynxtools_spm.nomad_uploader.uploader import (
+    run_uploader_with,
+    NOMADSettings,
+    DataProcessingSettings,
+)
+from pathlib import Path
+
+current_dir = Path(__file__).resolve().parent
+
+nomad_settings = NOMADSettings(
+    url_protocol="https",
+    url_domain="nomad-lab.eu",
+    url_version="prod/v1/oasis-b/api/v1/",
+    url="https://nomad-lab.eu/prod/v1/oasis-b/api/v1/",
+    username="Mozumder",
+    password="",
+    token="",
+    modify_upload_metadata=False,
+    publish_to_nomad=False,
+)
+local_src_dir = Path(
+    "/home/rubel/NOMAD-FAIRmat/nomad-distro-dev-RM/packages/pynxtools-spm/tests/data/nanonis"
+)
+total_upload = 3
+data_processing_settings = DataProcessingSettings(
+    raw_file_exts=(
+        ".dat",
+        ".sxm",
+    ),
+    single_batch_processing_time=total_upload * 90,  # seconds
+    src_dir=Path("/home/rubel/NOMAD-FAIRmat/SPMfolder/DataFilesForUpload"),
+    # copy_file_elsewhere=False,
+    dst_dir="",
+    create_pseudo_file=True,
+    pseudo_exts=".done",
+    spm_params_obj_l=[],
+    sts_eln=local_src_dir / "sts/version_gen_5e_with_described_nxdata/eln_data.yaml",
+    sts_config="",
+    stm_eln=local_src_dir / "stm/version_gen_4_5_with_described_nxdata/eln_data.yaml",
+    stm_config="",
+    afm_eln=local_src_dir / "afm/version_gen_4_with_described_nxdata/eln_data.yaml",
+    afm_config="",
+    logger_dir=current_dir,
+    number_of_uploads=total_upload,
+)
+
+if __name__ == "__main__":
+    run_uploader_with(
+        data_settings=nomad_settings,
+        data_processing_settings=data_processing_settings,
+    )

src/pynxtools_spm/nomad_uploader/files_movers.py

@@ -46,6 +46,7 @@ def copy_directory_structure(
     dst: Path,
     extension: Optional[str] = "",
     run_action_on_files: Optional[Callable] = None,
+    **args,
 ):
     """
     Copies a directory structure from source to destination directory
@@ -56,6 +57,7 @@ def copy_directory_structure(
     :param extension: File extension to filter by
     :param run_action_on_files: Function to run on each file such as modify files,
                       storing file path to an object etc.
+    :param args: Additional arguments to pass to the function run_action_on_files
     """
 
     if not src.is_dir():
@@ -78,5 +80,4 @@ def copy_directory_structure(
                 target_file_path.parent.mkdir(parents=True, exist_ok=True)
                 shutil.copy2(source_file, target_file_path)
                 if run_action_on_files is not None and callable(run_action_on_files):
-                    run_action_on_files(target_file_path)
-
+                    run_action_on_files(target_file_path, **args)

src/pynxtools_spm/nomad_uploader/helper.py

@@ -1,11 +1,13 @@
 import logging
 from typing import Union
 
-def setup_logger(name: str, 
-                 log_file: str, 
-                 level: Union[int, str] = logging.INFO, 
-                 existing_logger: logging.Logger = None
-                 ) -> tuple[logging.Logger, logging.FileHandler]:
+
+def setup_logger(
+    name: str,
+    log_file: str,
+    level: Union[int, str] = logging.INFO,
+    existing_logger: logging.Logger = None,
+) -> tuple[logging.Logger, logging.FileHandler]:
     """Set up a named logger that writes to a specific file.
     if existing_logger is provided, it will be used instead of creating a new one,
     and another handler will be added to it.

src/pynxtools_spm/nomad_uploader/nomad_upload_api.py

@@ -1,9 +1,23 @@
-import os
 import requests
+import logging
 
+from pathlib import Path
 
-def get_authentication_token(nomad_url, username, password):
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+handler = logging.StreamHandler()
+formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+handler.setFormatter(formatter)
+
+
+def get_authentication_token(
+    nomad_url: str, username: str, password: str, upload_logger: logging.Logger = None
+):
     """Get the token for accessing your NOMAD unpublished uploads remotely"""
+    if upload_logger is None:
+        upload_logger = logger
+
     try:
         response = requests.get(
             nomad_url + "auth/token",
@@ -12,18 +26,23 @@ def get_authentication_token(nomad_url, username, password):
         )
         token = response.json().get("access_token")
         if token:
+            logger.info("Successfully retrieved authentication token")
             return token
-
-        print("response is missing token: ")
-        print(response.json())
+        logger.error("Authentication token not found in response")
+        logger.error("Response: " + str(response.json()))
         return
-    except Exception:
-        print("something went wrong trying to get authentication token")
+    except Exception as e:
+        upload_logger.error(f"Something went wrong trying to get authentication token.")
+        upload_logger.error(f"Error in dataset creation: {e}")
         return
 
 
-def create_dataset(nomad_url, token, dataset_name):
+def create_dataset(
+    nomad_url: str, token: str, dataset_name: str, upload_logger: logging.Logger = None
+):
     """Create a dataset to group a series of NOMAD entries"""
+    if upload_logger is None:
+        upload_logger = logger
     try:
         response = requests.post(
             nomad_url + "datasets/",
@@ -34,23 +53,34 @@ def create_dataset(nomad_url, token, dataset_name):
         dataset_id = response.json().get("dataset_id")
         if dataset_id:
             return dataset_id
-
-        print("response is missing dataset_id: ")
-        print(response.json())
+        upload_logger.error("Dataset ID not found in response")
+        upload_logger.error("Response: " + str(response.json()))
         return
-    except Exception:
-        print("something went wrong trying to create a dataset")
+    except Exception as e:
+        upload_logger.error("Something went wrong trying to create a dataset")
+        upload_logger.error(f"Error in dataset creation: {e}")
         return
 
 
-def upload_to_NOMAD(nomad_url, token, upload_file):
+def upload_to_NOMAD(
+    nomad_url: str,
+    token: str,
+    upload_file: Path,
+    file_name: str = None,
+    upload_name: str = None,
+    upload_logger: logging.Logger = None,
+):
     """Upload a single file as a new NOMAD upload. Compressed zip/tar files are
     automatically decompressed.
     """
+    upload_name = upload_name if upload_name else upload_file.name
+    file_name = file_name if file_name else upload_file.name
+    if upload_logger is None:
+        upload_logger = logger
     with open(upload_file, "rb") as f:
         try:
             response = requests.post(
-                f"{nomad_url}uploads?file_name={os.path.basename(upload_file)}",
+                f"{nomad_url}uploads?file_name={file_name}&upload_name={upload_name}",
                 headers={
                     "Authorization": f"Bearer {token}",
                     "Accept": "application/json",
@@ -60,34 +90,73 @@ def upload_to_NOMAD(nomad_url, token, upload_file):
             )
             upload_id = response.json().get("upload_id")
             if upload_id:
+                logger.info(
+                    f"Successfully uploaded {upload_file} to NOMAD with ID {upload_id}"
+                )
                 return upload_id
 
-            print("response is missing upload_id: ")
-            print(response.json())
+            logger.error("Upload ID not found in response")
+            logger.error("Response: " + str(response.json()))
             return
         except Exception:
-            print("something went wrong uploading to NOMAD")
+            upload_logger.error(
+                f"Something went wrong uploading {upload_file} to NOMAD"
+            )
+            upload_logger.error(f"Error in upload: {upload_file}")
             return
 
-def trigger_reprocess_upload(nomad_url, token, upload_id):
+
+def trigger_reprocess_upload(
+    nomad_url: str, token: str, upload_id: str, upload_logger: logging.Logger = None
+):
     """Trigger reprocessing of an upload"""
+    if upload_logger is None:
+        upload_logger = logger
     try:
         response = requests.post(
             f"{nomad_url}uploads/{upload_id}/action/process",
             headers={"Authorization": f"Bearer {token}", "Accept": "application/json"},
             timeout=30,
         )
         return response
-    except Exception:
-        print("something went wrong trying to reprocess upload: " + upload_id)
+    except Exception as e:
+        upload_logger.error(
+            "Something went wrong trying to reprocess upload: " + upload_id
+        )
+        upload_logger.error(f"Error in reprocess: {e}")
+        return
+
+
+def delete_upload(
+    nomad_url: str, token: str, upload_id: str, upload_logger: logging.Logger = None
+):
+    """Delete an upload"""
+    if upload_logger is None:
+        upload_logger = logger
+    try:
+        response = requests.delete(
+            nomad_url + "uploads/" + upload_id,
+            headers={"Authorization": f"Bearer {token}", "Accept": "application/json"},
+            timeout=30,
+        )
+        return response
+    except Exception as e:
+        upload_logger.error(
+            "Something went wrong trying to delete upload: " + upload_id
+        )
+        upload_logger.error(f"Error in delete: {e}")
         return
 
 
-def check_upload_status(nomad_url, token, upload_id):
+def check_upload_status(
+    nomad_url: str, token: str, upload_id: str, upload_logger: logging.Logger = None
+):
     """
     # upload success => returns 'Process publish_upload completed successfully'
     # publish success => 'Process publish_upload completed successfully'
     """
+    if upload_logger is None:
+        upload_logger = logger
     try:
         response = requests.get(
             nomad_url + "uploads/" + upload_id,
@@ -98,16 +167,23 @@ def check_upload_status(nomad_url, token, upload_id):
         if status_message:
             return status_message
 
-        print("response is missing status_message: ")
-        print(response.json())
+        upload_logger.error("Upload status message not found in response")
+        upload_logger.error("Response: " + str(response.json()))
         return
-    except Exception:
-        print("something went wrong trying to check the status of upload" + upload_id)
-        # upload gets deleted from the upload staging area once published...or in this case something went wrong
+    except Exception as e:
+        upload_logger.error(
+            "Something went wrong trying to check the status of upload: " + upload_id
+        )
         return
 
 
-def edit_upload_metadata(nomad_url, token, upload_id, metadata):
+def edit_upload_metadata(
+    nomad_url: str,
+    token: str,
+    upload_id: str,
+    metadata: dict,
+    upload_logger: logging.Logger = None,
+):
     """
     Example of new metadata:
     upload_name = 'Test_Upload_Name'
@@ -122,7 +198,8 @@ def edit_upload_metadata(nomad_url, token, upload_id, metadata):
         },
     }
     """
-
+    if upload_logger is None:
+        upload_logger = logger
     try:
         response = requests.post(
             nomad_url + "uploads/" + upload_id + "/edit",
@@ -132,19 +209,29 @@ def edit_upload_metadata(nomad_url, token, upload_id, metadata):
         )
         return response
     except Exception:
-        print("something went wrong trying to add metadata to upload" + upload_id)
+        upload_logger.error(
+            "Something went wrong trying to edit metadata of upload: " + upload_id
+        )
+        upload_logger.error(f"Error in metadata edit: {upload_id}")
         return
 
 
-def publish_upload(nomad_url, token, upload_id):
+def publish_upload(
+    nomad_url: str, token: str, upload_id: str, upload_logger: logging.Logger = None
+):
     """Publish an upload"""
+    if upload_logger is None:
+        upload_logger = logger
     try:
         response = requests.post(
             nomad_url + "uploads/" + upload_id + "/action/publish",
             headers={"Authorization": f"Bearer {token}", "Accept": "application/json"},
             timeout=30,
         )
         return response
-    except Exception:
-        print("something went wrong trying to publish upload: " + upload_id)
+    except Exception as e:
+        upload_logger.error(
+            "Something went wrong trying to publish upload: " + upload_id
+        )
+        upload_logger.error(f"Error in publish: {e}")
         return