Update README.md

RubelMozumder · RubelMozumder · commit ade088fd6dcb · 2026-02-06T18:31:00.000+01:00
diff --git a/src/pynxtools_spm/nomad_uploader/README.md b/src/pynxtools_spm/nomad_uploader/README.md
@@ -135,7 +135,8 @@ if __name__ == "__main__":
 | `number_of_uploads` | int | No | 10 | Max files to process per batch |
 | `delete_failed_uploads` | bool | No | False | Delete uploads on timeout |
 | `upload_metadata` | dict | No | None | Metadata to apply to all uploads |
-| `file_specific_eln` | dict | No | None | Map filenames to specific ELN files |
+| `file_to_convert_data` | dict | No | None | Map file paths to ELN files and technique types |
+| `single_file_pynx_convert_time` | int | No | 5 | Timeout for single file conversion (seconds) |
 
 ## Metadata Management
 
@@ -254,6 +255,32 @@ Solution: Increase `max_upload_attempt` or `nomad_processing_time` in NOMADSetti
 
 ## Advanced Usage
 
+### Custom File-Specific Configuration
+
+Map specific files to custom ELN files and technique types using `file_to_convert_data`:
+
+```python
+data_proc_settings = DataProcessingSettings(
+    # ... other settings ...
+    file_to_convert_data={
+        "/path/to/data/stm/sample1.sxm": {
+            "eln": "/path/to/custom_eln1.yaml",  # Optional: use specific ELN
+            "technique": "stm",  # Required: specify technique (stm/sts/afm)
+        },
+        "/path/to/data/sts/measurement.dat": {
+            "eln": "",  # Empty: use default ELN from sts_eln setting
+            "technique": "sts",
+        },
+        "/path/to/data/afm/surface.sxm": {
+            "eln": "/path/to/custom_eln2.yaml",
+            "technique": "afm",
+        },
+    }
+)
+```
+
+**Important**: Use full file paths as keys to avoid collisions when files in different directories have the same name. The `__post_init__` method automatically creates both full-path and filename lookups.
+
 ### Processing Only Unprocessed Files
 
 The uploader automatically creates `.done` marker files for successful uploads. On subsequent runs, it only processes files without corresponding `.done` markers.
@@ -266,14 +293,52 @@ The uploader automatically creates `.done` marker files for successful uploads.
 # Only processes new files without .done markers
 ```
 
+### Automatic Technique Detection
+
+The uploader uses a hybrid approach to determine the SPM technique:
+
+1. **Explicit Configuration**: If `file_to_convert_data` specifies a `technique` for a file, it uses that value
+2. **File Extension Fallback**: If no explicit technique is configured:
+   - `.dat` files → STS (Scanning Tunneling Spectroscopy)
+   - `.sxm` files → STM (Scanning Tunneling Microscopy) by default
+
+**Best Practice**: Use `file_to_convert_data` to explicitly specify techniques, especially when:
+- `.sxm` files should be processed as AFM instead of STM
+- You have multiple files with the same name in different directories
+- You want to override default technique detection
+
+### Zip File Creation
+
+Each successful conversion creates a zip file containing:
+- **NeXus output file** (`.nxs`)
+- **Original raw data file** (`.dat` or `.sxm`)
+- **ELN metadata file** (`.yaml`)
+- **Config file** (if specified)
+
+The zip file uses basename extraction to ensure clean file names in the archive.
+
 ### Batch Processing with Custom ELN Mapping
 
+**Deprecated**: The old `file_specific_eln` parameter has been replaced with `file_to_convert_data`.
+
 ```python
-data_proc_settings.file_specific_eln = {
-    "sample1.dat": Path("/path/to/sample1_eln.yaml"),
-    "sample2.sxm": Path("/path/to/sample2_eln.yaml"),
+# Old approach (deprecated):
+# data_proc_settings.file_specific_eln = {
+#     "sample1.dat": Path("/path/to/sample1_eln.yaml"),
+# }
+
+# New approach (recommended):
+data_proc_settings.file_to_convert_data = {
+    "/full/path/to/sample1.dat": {
+        "eln": "/path/to/sample1_eln.yaml",
+        "technique": "sts"
+    },
+    "/full/path/to/sample2.sxm": {
+        "eln": "/path/to/sample2_eln.yaml", 
+        "technique": "afm"  # Explicitly specify AFM for .sxm file
+    },
 }
-# Default ELN will be used for files not in this mapping
+# Default ELN (sts_eln, stm_eln, afm_eln) used for files not in this mapping
 ```
 
 ### Publishing to NOMAD
@@ -329,9 +394,18 @@ See `example_upload_script.py` for a complete working example with real configur
 
 When modifying the uploader:
 1. Update type hints (`Optional[Literal[...]]` for restricted values)
-2. Maintain consistent logger usage (pass `upload_logger` to all functions)
-3. Add comprehensive docstrings
-4. Update this README with new features
+2. Maintain consistent logger usage (pass `upload_logger` and `converter_logger` to functions)
+3. Use `Path.name` for extracting filenames instead of string splitting
+4. Add comprehensive docstrings
+5. Update this README with new features
+6. Test with various file types (.dat, .sxm) and techniques (STS, STM, AFM)
+
+### Recent Changes
+
+- **v2.0**: Replaced `file_specific_eln` with `file_to_convert_data` for better technique specification
+- **Improved**: File path handling using `Path.name` instead of string manipulation
+- **Enhanced**: Automatic technique detection with explicit configuration support
+- **Fixed**: Collision handling when multiple files have the same name in different directories
 
 
 ## Support
diff --git a/src/pynxtools_spm/nomad_uploader/example_upload_script.py b/src/pynxtools_spm/nomad_uploader/example_upload_script.py
@@ -6,6 +6,7 @@
 from pathlib import Path
 
 current_dir = Path(__file__).resolve().parent
+project_dir = current_dir.parent.parent.parent
 
 nomad_settings = NOMADSettings(
     url_protocol="https",
@@ -14,35 +15,100 @@
     url="https://nomad-lab.eu/prod/v1/develop/api/v1/",
     username="Mozumder",
     password="*#R516660a*#",
-    token="eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJmb1hmZnM5QlFQWHduLU54Yk5PYlExOFhnZnlKU1FNRkl6ZFVnWjhrZzdVIn0.eyJleHAiOjE3NzAzODExMzMsImlhdCI6MTc3MDI5NDczMywianRpIjoiYjk2YjI1NTUtYmU4My00Mjk0LWFlMDMtOTFjMTNmN2RlNmMwIiwiaXNzIjoiaHR0cHM6Ly9ub21hZC1sYWIuZXUvZmFpcmRpL2tleWNsb2FrL2F1dGgvcmVhbG1zL2ZhaXJkaV9ub21hZF9wcm9kIiwic3ViIjoiOGVjOGMwOWUtOWZiMC00YTQ3LTllNTEtNDRkNWFjNjg5YWQwIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoibm9tYWRfcHVibGljIiwic2lkIjoiYjZlODVlZmUtMzViOS00M2UzLTgxMWItOWU3ZWZlMTc4MTVkIiwic2NvcGUiOiJvcGVuaWQgcHJvZmlsZSBlbWFpbCIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJuYW1lIjoiUnViZWwgTW96dW1kZXIiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJtb3p1bWRlciIsInNlc3Npb25fc3RhdGUiOiJiNmU4NWVmZS0zNWI5LTQzZTMtODExYi05ZTdlZmUxNzgxNWQiLCJnaXZlbl9uYW1lIjoiUnViZWwiLCJmYW1pbHlfbmFtZSI6Ik1venVtZGVyIiwiZW1haWwiOiJtb3p1bWRlckBwaHlzaWsuaHUtYmVybGluLmRlIn0.ekOAgc5AIqm_uriTRj8yxt-Kmtuz5IHf6Bs3lQIaW0WK_ds7UD21itBtKFKTZlUHq2Ti6gcfsU1WqkdRJ3_5d1xSblJoKWL974o8YuAhDMNLkcv4HqthjQ_pnp3XSb_y0JXsPB0tOImjf-86sLqYqsgn9FHcln-OZb73guLlrKiwyXt4cK5pB-vO8JFKdZvChnVHhb7wyBYkJuLYtvhjQgASLDqDSAPHm6mk9ZGtqcr1KtzlopMzK2YovfgNiJSWrKyH6yI2O4wcSJzc6374N3SAQeIClYGExs0f39jKS0dfhdROI753SUvDAO-niDiyxT8LChwHtuM1IlnFSw-5ng",
+    token="",
     modify_upload_metadata=True,
     publish_to_nomad=False,
 )
-local_src_dir = Path(
-    "/home/rubel/NOMAD-FAIRmat/nomad-distro-dev-RM/packages/pynxtools-spm/tests/data/nanonis"
-)
-total_upload = 1
+nanonis = project_dir / "tests/data/nanonis"
+
+total_upload = 3
+
 data_proc_settings = DataProcessingSettings(
     raw_file_exts=(
         ".dat",
         ".sxm",
     ),
     single_batch_processing_time=total_upload * 90,  # seconds
-    src_dir=local_src_dir,  # Path("/home/rubel/NOMAD-FAIRmat/SPMfolder/DataFilesForUpload"),
+    src_dir=nanonis,
     # copy_file_elsewhere=False,
     dst_dir="",
     create_pseudo_file=True,
     pseudo_exts=".done",
     spm_params_obj_l=[],
-    sts_eln=local_src_dir / "sts/version_gen_5e_with_described_nxdata/eln_data.yaml",
+    sts_eln=nanonis / "sts/version_gen_5e_with_described_nxdata/eln_data.yaml",
     sts_config="",
-    stm_eln=local_src_dir / "stm/version_gen_5_with_described_nxdata/eln_data.yaml",
+    stm_eln=nanonis / "stm/version_gen_5_with_described_nxdata/eln_data.yaml",
     stm_config="",
-    afm_eln=local_src_dir / "afm/version_gen_4_with_described_nxdata/eln_data.yaml",
+    afm_eln=nanonis / "afm/version_gen_4_with_described_nxdata/eln_data.yaml",
     afm_config="",
     logger_dir=current_dir,
     number_of_uploads=total_upload,
-    file_specific_eln={"raw_file_name": "eln_file_path"},
+    file_to_convert_data={
+        f"{nanonis / 'stm/version_gen_5_with_described_nxdata/Au_mica_2023_Y_A_diPAMY_195.sxm'}": {
+            "eln": "",
+            "technique": "stm",
+        },
+        f"{nanonis / 'sts/version_gen_5e_with_described_nxdata/STS_nanonis_generic_5e_1.dat'}": {
+            "eln": "",
+            "technique": "sts",
+        },
+        f"{nanonis / 'stm/version_gen_5_with_described_nxdata/Au_mica_2023_Y_A_diPAMY_195.sxm'}": {
+            "eln": "",
+            "technique": "stm",
+        },
+        f"{nanonis / 'sts/version_gen_5e_with_described_nxdata/STS_nanonis_generic_5e_1.dat'}": {
+            "eln": "",
+            "technique": "sts",
+        },
+        f"{nanonis / 'stm/version_gen_5_with_described_nxdata/Au_mica_2023_Y_A_diPAMY_195.sxm'}": {
+            "eln": "",
+            "technique": "stm",
+        },
+        f"{nanonis / 'sts/version_gen_5e_with_described_nxdata/STS_nanonis_generic_5e_1.dat'}": {
+            "eln": "",
+            "technique": "sts",
+        },
+        f"{nanonis / 'stm/version_gen_5_with_described_nxdata/Au_mica_2023_Y_A_diPAMY_195.sxm'}": {
+            "eln": "",
+            "technique": "stm",
+        },
+        f"{nanonis / 'sts/version_gen_5e_with_described_nxdata/STS_nanonis_generic_5e_1.dat'}": {
+            "eln": "",
+            "technique": "sts",
+        },
+        f"{nanonis / 'stm/version_gen_5_with_described_nxdata/Au_mica_2023_Y_A_diPAMY_195.sxm'}": {
+            "eln": "",
+            "technique": "stm",
+        },
+        f"{nanonis / 'sts/version_gen_5e_with_described_nxdata/STS_nanonis_generic_5e_1.dat'}": {
+            "eln": "",
+            "technique": "sts",
+        },
+        f"{nanonis / 'stm/version_gen_5_with_described_nxdata/Au_mica_2023_Y_A_diPAMY_195.sxm'}": {
+            "eln": "",
+            "technique": "stm",
+        },
+        f"{nanonis / 'sts/version_gen_5e_with_described_nxdata/STS_nanonis_generic_5e_1.dat'}": {
+            "eln": "",
+            "technique": "sts",
+        },
+        f"{nanonis / 'stm/version_gen_5_with_described_nxdata/Au_mica_2023_Y_A_diPAMY_195.sxm'}": {
+            "eln": "",
+            "technique": "stm",
+        },
+        f"{nanonis / 'sts/version_gen_5e_with_described_nxdata/STS_nanonis_generic_5e_1.dat'}": {
+            "eln": "",
+            "technique": "sts",
+        },
+        f"{nanonis / 'stm/version_gen_5_with_described_nxdata/Au_mica_2023_Y_A_diPAMY_195.sxm'}": {
+            "eln": "",
+            "technique": "stm",
+        },
+        f"{nanonis / 'sts/version_gen_5e_with_described_nxdata/STS_nanonis_generic_5e_1.dat'}": {
+            "eln": "",
+            "technique": "sts",
+        },
+    },
     #     metadata = {
     #     "metadata": {
     #     "upload_name": upload_name,
diff --git a/src/pynxtools_spm/nomad_uploader/reader_config_setup.py b/src/pynxtools_spm/nomad_uploader/reader_config_setup.py
@@ -55,15 +55,14 @@ def convert_spm_experiments(
     if not input_params.expriement_type:
         converter_logger.error("Experiment type is required to run an SPM experiment")
 
-    input_params.input_file = (*input_params.input_file, input_params.eln)
     input_params.input_file = tuple(
-        Path(file) if isinstance(file, str) else file
-        for file in input_params.input_file
+        map(Path, (*input_params.input_file, input_params.eln))
     )
+
     if input_params.config:
         input_params.input_file = (
             *input_params.input_file,
-            input_params.config,
+            Path(input_params.config),
         )
 
     zip_file = None
@@ -88,24 +87,22 @@ def convert_spm_experiments(
         converter_logger.addHandler(converter_handeler)
     try:
         kwargs = asdict(input_params)
-        print("#### kwargs:", kwargs)
         kwargs["input_file"] = tuple(map(str, input_params.input_file))
         kwargs["output"] = str(input_params.output)
         # with converter_logger:
         convert(**kwargs)
-        print("#### kwargs after conversion:", kwargs)
         if input_params.create_zip:
             with zipfile.ZipFile(zip_file, "w") as zipf:
                 zipf.write(
                     str(input_params.output),
-                    arcname=str(input_params.output).split("/")[-1],
+                    arcname=Path(input_params.output).name,
                 )
                 for file in map(str, input_params.input_file):
-                    zipf.write(file, arcname=file.split("/")[-1])
+                    zipf.write(file, arcname=Path(file).name)
             input_params.zip_file_path = Path(zip_file)
 
     except Exception as e:
-        print("NeXusConverterError:", e)
+        converter_logger.error(f"Error: {e}")
     finally:
         # Prevent propagatting other logs through this handler
         converter_logger.removeHandler(converter_handeler)
diff --git a/src/pynxtools_spm/nomad_uploader/uploader.py b/src/pynxtools_spm/nomad_uploader/uploader.py
@@ -87,11 +87,19 @@ class DataProcessingSettings:
     number_of_uploads: int = 10
     delete_failed_uploads: bool = False
     upload_metadata: Optional[dict] = None
-    # Raw file to ELN file mapping.
-    file_specific_eln: Optional[dict] = None
+    # File to converter data
+    # file_to_converter_data = {'file_name': {'eln': 'eln_file_path',
+    #                                         'technique': 'stm/sts/afm'}}
+    file_to_convert_data: Optional[dict] = None
     # Time for individual file conversion
     single_file_pynx_convert_time: int = 5  # seconds
 
+    def __post_init__(self):
+        file_obj = {}
+        for file, obj in self.file_to_convert_data.items():
+            file_obj[Path(file).name] = obj
+        self.file_to_convert_data = file_obj
+
 
 def create_preseudo_file(
     params_obj: SPMConvertInputParameters,
@@ -132,9 +140,11 @@ def get_unprocessed_files(src_dir: Path, data_proc_settings) -> list:
     file is not present.
     """
     process_status_map = {}
+    # Collect all raw files
     for file in src_dir.glob("**/*.*"):
         if file.is_file() and file.suffix in data_proc_settings.raw_file_exts:
             process_status_map[file] = False
+    # Mark processed files as True
     for file in src_dir.glob("**/*.*"):
         if file.is_file() and file.suffix == data_proc_settings.pseudo_exts:
             # Remove extra pseudo extension
@@ -156,10 +166,17 @@ def set_and_store_prepared_parameters(
     spm_tech: Optional[Literal["stm", "sts", "afm"]] = None,
 ) -> None:
     params_obj = None
-    spec_eln_file = data_proc_settings.file_specific_eln.get(
-        file.name, data_proc_settings.sts_eln
+    spec_eln_file = (
+        data_proc_settings.file_to_convert_data.get(file.name, {}).get("eln")
+        if data_proc_settings.file_to_convert_data
+        else None
     )
-    if file.suffix == ".dat":
+    technique = (
+        data_proc_settings.file_to_convert_data.get(file.name, {}).get("technique")
+        if data_proc_settings.file_to_convert_data
+        else None
+    )
+    if technique == "sts" or file.suffix == ".dat":
         params_obj = SPMConvertInputParameters(
             input_file=(file,),
             eln=spec_eln_file if spec_eln_file else data_proc_settings.sts_eln,
@@ -169,7 +186,7 @@ def set_and_store_prepared_parameters(
             raw_extension="dat",
             create_zip=True,
         )
-    elif file.suffix == ".sxm":
+    elif technique == "stm" or file.suffix == ".sxm":
         params_obj = SPMConvertInputParameters(
             input_file=(file,),
             eln=spec_eln_file if spec_eln_file else data_proc_settings.stm_eln,
@@ -179,7 +196,7 @@ def set_and_store_prepared_parameters(
             raw_extension="sxm",
             create_zip=True,
         )
-    elif file.suffix == ".sxm":
+    elif technique == "afm" or file.suffix == ".sxm":
         params_obj = SPMConvertInputParameters(
             input_file=(file,),
             eln=spec_eln_file if spec_eln_file else data_proc_settings.afm_eln,
@@ -351,8 +368,9 @@ def queue_results(input_params, lock, results_q):
                 upload_id = upload_to_NOMAD(
                     nomad_settings.url, nomad_settings.token, zip_to_upload
                 )
+
                 upload_logger.info(
-                    f"Upload request with Upload ID ({upload_id}) corresponding to {complete_param_obj.input_file}."
+                    f"Upload request with Upload ID ({upload_id}) corresponding to files \n{'\n'.join(map(str, complete_param_obj.input_file))}."
                 )
                 # trigger_reprocess_upload(
                 #     nomad_settings.url, nomad_settings.token, upload_id
