fix: filter out observations without a value (#1851)

Author: mschoettle

opal/services/fhir/ips.py

@@ -27,6 +27,19 @@
 from fhir.resources.R4B.reference import Reference
 
 
+def _clean_observations(observations: list[Observation]) -> list[Observation]:
+    """
+    Clean up observations.
+
+    Remove those:
+        - without a category
+        - without a value and therefore a dataAbsentReason
+    """  # noqa: DOC201
+    return [
+        observation for observation in observations if observation.category and observation.dataAbsentReason is None
+    ]
+
+
 def build_patient_summary(  # noqa: PLR0913, PLR0917
     patient: Patient,
     conditions: list[Condition],
@@ -52,18 +65,13 @@ def build_patient_summary(  # noqa: PLR0913, PLR0917
     # If a language is configured for IPS, use it to generate the bundle
     # Otherwise, default to the system's primary language (assuming clinical data is typically saved in this language)
     ips_language = settings.IPS_LANGUAGE or settings.LANGUAGES[0][0]
-
-    observations_with_category = [observation for observation in observations if observation.category]
+    cleaned_observations = _clean_observations(observations)
 
     vital_signs = [
-        observation
-        for observation in observations_with_category
-        if observation.category[0].coding[0].code == 'vital-signs'
+        observation for observation in cleaned_observations if observation.category[0].coding[0].code == 'vital-signs'
     ]
     labs = [
-        observation
-        for observation in observations_with_category
-        if observation.category[0].coding[0].code == 'laboratory'
+        observation for observation in cleaned_observations if observation.category[0].coding[0].code == 'laboratory'
     ]
 
     # Translates static strings in the right language for the bundle

opal/services/fhir/tests/fixtures/observations.json

@@ -376,6 +376,51 @@
                     ]
                 }
             }
+        },
+        {
+            "fullUrl": "urn:uuid:9efb0a09-01e2-4792-8085-439698be315e",
+            "resource": {
+                "resourceType": "Observation",
+                "id": "9efb0a09-01e2-4792-8085-439698be315e",
+                "meta": {
+                    "versionId": "1",
+                    "lastUpdated": "2025-10-30T11:47:52-05:00"
+                },
+                "status": "final",
+                "category": [
+                    {
+                        "coding": [
+                            {
+                                "system": "http://terminology.hl7.org/CodeSystem/observation-category",
+                                "code": "vital-signs"
+                            }
+                        ]
+                    }
+                ],
+                "code": {
+                    "coding": [
+                        {
+                            "system": "http://loinc.org",
+                            "code": "3151-8",
+                            "display": "Inhaled oxygen flow rate"
+                        }
+                    ]
+                },
+                "subject": {
+                    "reference": "Patient/9ef97180-cd66-4fb3-a903-f1b78bd693a5",
+                    "type": "Patient"
+                },
+                "effectiveDateTime": "2025-10-10T10:27:03-05:00",
+                "dataAbsentReason": {
+                    "coding": [
+                        {
+                            "system": "http://terminology.hl7.org/CodeSystem/data-absent-reason",
+                            "code": "unknown",
+                            "display": "Unknown"
+                        }
+                    ]
+                }
+            }
         }
     ]
 }

opal/services/fhir/tests/test_fhir.py

@@ -388,7 +388,7 @@ def test_patient_observations(self, fhir_connector: FHIRConnector, mocker: Mocke
 
         observations = fhir_connector.patient_observations('test-patient-uuid')
 
-        assert len(observations) == 9
+        assert len(observations) == 10
         assert observations[0].id == '59ace158-3be6-11f0-9645-fa163e09c13a'
         assert observations[1].id == '59acedd7-3be6-11f0-9645-fa163e09c13a'
         fhir_connector.session.get.assert_called_once_with(
@@ -403,7 +403,7 @@ def test_patient_observations_no_category(self, fhir_connector: FHIRConnector, m
 
         observations = fhir_connector.patient_observations('test-patient-uuid')
 
-        assert len(observations) == 9
+        assert len(observations) == 10
         assert observations[8].id == 'a083c331-bd33-4372-8c4d-8c329d354607'
         assert observations[8].category is None
 

opal/services/fhir/tests/test_ips.py

@@ -78,15 +78,15 @@ def test_build_patient_summary() -> None:
     assert summary.type == 'document'
     assert summary.identifier.system == 'urn:oid:2.16.724.4.8.10.200.10'
 
-    observations_with_category = [observation for observation in observations if observation.category]
+    observations_with_category_value = ips._clean_observations(observations)
 
     # Composition, Patient, Device and the resources
     assert len(summary.entry) == (
         3
         + len(conditions)
         + len(medication_requests)
         + len(allergies)
-        + len(observations_with_category)
+        + len(observations_with_category_value)
         + len(immunizations)
     )
 
@@ -106,16 +106,16 @@ def test_build_patient_summary_composition() -> None:
     # Verify all 7 IPS sections
     assert len(composition.section) == 7
 
-    observations_with_category = [observation for observation in observations if observation.category]
+    observations_with_category_value = ips._clean_observations(observations)
 
     vital_signs = [
         observation
-        for observation in observations_with_category
+        for observation in observations_with_category_value
         if observation.category[0].coding[0].code == 'vital-signs'
     ]
     labs = [
         observation
-        for observation in observations_with_category
+        for observation in observations_with_category_value
         if observation.category[0].coding[0].code == 'laboratory'
     ]
 
@@ -209,7 +209,7 @@ def test_build_patient_summary_resources_included() -> None:
     expected_ids.update(condition.id for condition in conditions)
     expected_ids.update(medication_request.id for medication_request in medication_requests)
     expected_ids.update(allergy.id for allergy in allergies)
-    expected_ids.update(observation.id for observation in observations if observation.category)
+    expected_ids.update(observation.id for observation in ips._clean_observations(observations))
     expected_ids.update(immunization.id for immunization in immunizations)
 
     assert resource_ids == expected_ids
@@ -225,10 +225,23 @@ def test_build_patient_summary_resources_observations_without_category() -> None
         _prepare_build_patient_summary()
     )
 
-    # Skip the first three entries (Composition, Patient, Device)
     resource_ids = {entry.resource.id for entry in summary.entry}
 
-    observations_without_category = [observation for observation in observations if not observation.category]
+    observations_without_category = [observation.id for observation in observations if not observation.category]
+
+    assert len(observations_without_category) == 1
+    assert 'a083c331-bd33-4372-8c4d-8c329d354607' in observations_without_category
+    assert 'a083c331-bd33-4372-8c4d-8c329d354607' not in resource_ids
+
+
+def test_build_patient_summary_resources_observations_without_data_absent_reason() -> None:
+    """Observations with a dataAbsentReason are not included in the patient summary Bundle."""
+    summary, _patient, _conditions, _medication_requests, _allergies, observations, _immunizations = (
+        _prepare_build_patient_summary()
+    )
+
+    resource_ids = {entry.resource.id for entry in summary.entry}
+    observations_without_value = [observation.id for observation in observations if observation.dataAbsentReason]
 
-    for observation in observations_without_category:
-        assert observation.id not in resource_ids
+    assert len(observations_without_value) == 1
+    assert '9efb0a09-01e2-4792-8085-439698be315e' not in resource_ids