added a check on the concept_value

Author: ChaoPang

src/cehrbert/data_generators/hf_data_generator/hf_dataset_mapping.py

@@ -582,8 +582,10 @@ def transform(self, record: Dict[str, Any]) -> Dict[str, Any]:
         if "concept_values" not in record:
             record["concept_values"] = record["number_as_values"]
 
-        if np.isnan(record["concept_values"]).any():
-            record["concept_values"] = [v if not pd.isna(v) else 0.0 for v in record["concept_values"]]
+        concept_value_is_nan = np.isnan(record["concept_values"])
+        if concept_value_is_nan.any():
+            concept_value_masks[concept_value_is_nan] = 0
+            record["concept_value_masks"] = concept_value_masks
 
         assert len(input_ids) == len(
             record["concept_ids"]

src/cehrbert/models/hf_models/tokenization_utils.py

@@ -42,7 +42,7 @@ def map_statistics(batch: Dict[str, Any], capacity=100, value_outlier_std=2.0) -
         for concept_id, concept_value, concept_value_indicator, unit in zip(
             concept_ids, concept_values, concept_value_indicators, units
         ):
-            if concept_value_indicator == 1:
+            if concept_value_indicator == 1 and concept_value:
                 numeric_stats_by_lab[(concept_id, unit)].add(1, concept_value)
     return {"numeric_stats_by_lab": numeric_stats_by_lab}