using readonly instead of user_editability; enabling injesting multip… (#486)

* using readonly instead of user_editability; enabling injesting multiple methods to the base classes and not only the normalization; Entry to inherit also from Task, so Workflow click drives there directly

* moving changes to NeXusMeasurement which is not anymore using normalisation from Measurement and Activity

* fix for the new normalisation function

* nicer name layout

* removing unneeded comments; fixing race condition at NeXusdataConverter ELN normalisation

* m_copy the relevant sections to steps and results instead of using python reference

* Fix attribute error.

* remove attribute checks.

---------

Co-authored-by: Rubel <rubel.mozumder@outlook.com>

Author: sanbrock

src/pynxtools/nomad/dataconverter.py

@@ -17,9 +17,7 @@
 from pynxtools.dataconverter import convert as pynxtools_converter
 from pynxtools.dataconverter import writer as pynxtools_writer
 from pynxtools.dataconverter.template import Template
-from pynxtools.definitions.dev_tools.utils.nxdl_utils import (
-    get_app_defs_names,  # pylint: disable=import-error
-)
+from pynxtools.definitions.dev_tools.utils.nxdl_utils import get_app_defs_names  # pylint: disable=import-error
 
 m_package = Package(name="nexus_data_converter")
 
@@ -96,6 +94,14 @@ def populate_nexus_subsection(
         archive.data.output = os.path.join(
             archive.m_context.raw_path(), output_file_path
         )
+        # remove the nexus file and ensure that NOMAD knows that it is removed
+        try:
+            os.remove(archive.data.output)
+            archive.m_context.process_updated_raw_file(
+                archive.data.output, allow_modify=True
+            )
+        except Exception as e:
+            pass
         pynxtools_writer.Writer(
             data=template, nxdl_f_path=nxdl_f_path, output_path=archive.data.output
         ).write()
@@ -220,6 +226,15 @@ def normalize(self, archive, logger):
             ],
             "output": os.path.join(raw_path, archive.data.output),
         }
+        # remove the nexus file and ensure that NOMAD knows that it is removed
+        try:
+            os.remove(os.path.join(raw_path, archive.data.output))
+            archive.m_context.process_updated_raw_file(
+                archive.data.output, allow_modify=True
+            )
+        except Exception as e:
+            pass
+        # create the new nexus file
         try:
             pynxtools_converter.logger = logger
             pynxtools_converter.helpers.logger = logger
@@ -229,7 +244,7 @@ def normalize(self, archive, logger):
                 "could not convert to nxs", mainfile=archive.data.output, exc_info=e
             )
             raise e
-
+        # parse the new nexus file
         try:
             archive.m_context.process_updated_raw_file(
                 archive.data.output, allow_modify=True

src/pynxtools/nomad/parser.py

@@ -513,7 +513,7 @@ def parse(
         if archive.metadata.entry_type is None:
             archive.metadata.entry_type = app_def
             archive.metadata.domain = "nexus"
-        archive.metadata.user_editable = False
+        archive.metadata.readonly = True
 
         # Normalise element info
         if archive.results is None:

src/pynxtools/nomad/schema.py

@@ -47,6 +47,7 @@
         InstrumentReference,
         Measurement,
     )
+    from nomad.datamodel.metainfo.workflow import Link, Task, Workflow
     from nomad.metainfo import (
         Attribute,
         Bytes,
@@ -118,11 +119,14 @@ def normalize(self, archive, logger):
             app_entry = getattr(self, "ENTRY")
             if len(app_entry) < 1:
                 raise AttributeError()
-            self.steps = app_entry
+            self.steps = []
             for entry in app_entry:
+                sec_c = entry.m_copy()
+                self.steps.append(sec_c)
                 for sec in entry.m_all_contents():
                     if isinstance(sec, ActivityStep):
-                        self.steps.append(sec)
+                        sec_c = sec.m_copy()
+                        self.steps.append(sec_c)
                     elif isinstance(sec, basesections.Instrument):
                         ref = InstrumentReference(name=sec.name)
                         ref.reference = sec
@@ -132,14 +136,52 @@ def normalize(self, archive, logger):
                         ref.reference = sec
                         self.samples.append(ref)
                     elif isinstance(sec, ActivityResult):
-                        self.results.append(sec)
+                        sec_c = sec.m_copy()
+                        self.results.append(sec_c)
             if self.m_def.name == "Root":
                 self.method = "Generic Experiment"
             else:
                 self.method = self.m_def.name + " Experiment"
         except (AttributeError, TypeError):
             pass
-        super(NexusMeasurement, self).normalize(archive, logger)
+        super(basesections.Activity, self).normalize(archive, logger)
+
+        if archive.results.eln.methods is None:
+            archive.results.eln.methods = []
+        if self.method:
+            archive.results.eln.methods.append(self.method)
+        else:
+            archive.results.eln.methods.append(self.m_def.name)
+        if archive.workflow2 is None:
+            archive.workflow2 = Workflow(name=self.name)
+        # steps to tasks
+        act_array = archive.workflow2.tasks
+        existing_items = {(task.name, task.section) for task in act_array}
+        new_items = [
+            item.to_task()
+            for item in self.steps
+            if (item.name, item) not in existing_items
+        ]
+        act_array.extend(new_items)
+        # samples to inputs
+        act_array = archive.workflow2.inputs
+        existing_items = {(link.name, link.section) for link in act_array}
+        new_items = [
+            Link(name=item.name, section=item.reference)
+            for item in self.samples
+            if (item.name, item.reference) not in existing_items
+        ]
+        act_array.extend(new_items)
+
+        # results to outputs
+        act_array = archive.workflow2.outputs
+        existing_items = {(link.name, link.section) for link in act_array}
+        new_items = [
+            Link(name=item.name, section=item)
+            for item in self.results
+            if (item.name, item) not in existing_items
+        ]
+        act_array.extend(new_items)
 
 
 VALIDATE = False
@@ -952,7 +994,7 @@ def normalize_sample(self, archive, logger):
     """Normalizer for sample section."""
     current_cls = __section_definitions[__rename_nx_for_nomad("NXsample")].section_cls
     self.name = self.__dict__["nx_name"] + (
-        "(" + self.name__field + ")" if self.name__field else ""
+        " (" + self.name__field + ")" if self.name__field else ""
     )
     # one could also copy local ids to identifier for search purposes
     super(current_cls, self).normalize(archive, logger)
@@ -964,7 +1006,7 @@ def normalize_entry(self, archive, logger):
     if self.start_time__field:
         self.start_time = self.start_time__field
     self.name = self.__dict__["nx_name"] + (
-        "(" + self.title__field + ")" if self.title__field is not None else ""
+        " (" + self.title__field + ")" if self.title__field is not None else ""
     )
     # one could also copy local ids to identifier for search purposes
     super(current_cls, self).normalize(archive, logger)
@@ -998,7 +1040,7 @@ def create_Entity(lab_id, archive, f_name):
             data=entitySec,
             m_context=archive.m_context,
             metadata=EntryMetadata(
-                entry_type="identifier", domain="nexus"
+                entry_type="identifier", domain="nexus", readonly=True
             ),  # upload_id=archive.m_context.upload_id,
         )
         with archive.m_context.raw_file(f_name, "w") as f_obj:
@@ -1039,8 +1081,12 @@ def get_entry_reference(archive, f_name):
     __rename_nx_for_nomad("NXsample"): normalize_sample,
     __rename_nx_for_nomad("NXsample_component"): normalize_sample_component,
     __rename_nx_for_nomad("NXidentifier"): normalize_identifier,
-    __rename_nx_for_nomad("NXentry"): normalize_entry,
-    __rename_nx_for_nomad("NXprocess"): normalize_process,
+    __rename_nx_for_nomad("NXentry"): {
+        "normalize": normalize_entry,
+    },
+    __rename_nx_for_nomad("NXprocess"): {
+        "normalize": normalize_process,
+    },
     __rename_nx_for_nomad("NXdata"): normalize_data,
 }
 
@@ -1053,4 +1099,8 @@ def get_entry_reference(archive, f_name):
 
     # Append the normalize method from a function
     if normalize_func:
-        section.section_cls.normalize = normalize_func
+        if isinstance(normalize_func, dict):
+            for key, value in normalize_func.items():
+                setattr(section.section_cls, key, value)
+        else:
+            section.section_cls.normalize = normalize_func