set None unit to a default value N/A

Author: ChaoPang

src/cehrbert/data_generators/hf_data_generator/hf_dataset_mapping.py

@@ -578,6 +578,11 @@ def transform(self, record: Dict[str, Any]) -> Dict[str, Any]:
         input_ids = self._concept_tokenizer.encode(record["concept_ids"])
         record["input_ids"] = input_ids
         concept_value_masks = record["concept_value_masks"]
+        units = record["units"]
+        if np.isnan(units).any():
+            units = record["units"].copy()
+            units[np.isnan(units)] = "N/A"
+
         # Backward compatibility
         if "concept_values" not in record:
             record["concept_values"] = record["number_as_values"]