Merge pull request #169 from CITCOM-project/ci_pos_neg

Confidence intervals through inheritence

Author: jmafoster1

causal_testing/testing/causal_test_outcome.py

@@ -22,7 +22,29 @@ def __str__(self) -> str:
         return type(self).__name__
 
 
-class ExactValue(CausalTestOutcome):
+class SomeEffect(CausalTestOutcome):
+    """An extension of TestOutcome representing that the expected causal effect should not be zero."""
+
+    def apply(self, res: CausalTestResult) -> bool:
+        if res.test_value.type == "ate":
+            return (0 < res.ci_low() < res.ci_high()) or (res.ci_low() < res.ci_high() < 0)
+        if res.test_value.type == "risk_ratio":
+            return (1 < res.ci_low() < res.ci_high()) or (res.ci_low() < res.ci_high() < 1)
+        raise ValueError(f"Test Value type {res.test_value.type} is not valid for this TestOutcome")
+
+
+class NoEffect(CausalTestOutcome):
+    """An extension of TestOutcome representing that the expected causal effect should be zero."""
+
+    def apply(self, res: CausalTestResult) -> bool:
+        if res.test_value.type == "ate":
+            return (res.ci_low() < 0 < res.ci_high()) or (abs(res.test_value.value) < 1e-10)
+        if res.test_value.type == "risk_ratio":
+            return (res.ci_low() < 1 < res.ci_high()) or np.isclose(res.test_value.value, 1.0, atol=1e-10)
+        raise ValueError(f"Test Value type {res.test_value.type} is not valid for this TestOutcome")
+
+
+class ExactValue(SomeEffect):
     """An extension of TestOutcome representing that the expected causal effect should be a specific value."""
 
     def __init__(self, value: float, tolerance: float = None):
@@ -33,6 +55,8 @@ def __init__(self, value: float, tolerance: float = None):
             self.tolerance = tolerance
 
     def apply(self, res: CausalTestResult) -> bool:
+        if res.ci_valid():
+            return super().apply(res) and np.isclose(res.test_value.value, self.value, atol=self.tolerance)
         return np.isclose(res.test_value.value, self.value, atol=self.tolerance)
 
     def __str__(self):
@@ -43,6 +67,8 @@ class Positive(CausalTestOutcome):
     """An extension of TestOutcome representing that the expected causal effect should be positive."""
 
     def apply(self, res: CausalTestResult) -> bool:
+        if res.ci_valid() and not super().apply(res):
+            return False
         if res.test_value.type == "ate":
             return res.test_value.value > 0
         if res.test_value.type == "risk_ratio":
@@ -54,36 +80,10 @@ class Negative(CausalTestOutcome):
     """An extension of TestOutcome representing that the expected causal effect should be negative."""
 
     def apply(self, res: CausalTestResult) -> bool:
+        if res.ci_valid() and not super().apply(res):
+            return False
         if res.test_value.type == "ate":
             return res.test_value.value < 0
         if res.test_value.type == "risk_ratio":
             return res.test_value.value < 1
         raise ValueError(f"Test Value type {res.test_value.type} is not valid for this TestOutcome")
-
-
-class SomeEffect(CausalTestOutcome):
-    """An extension of TestOutcome representing that the expected causal effect should not be zero."""
-
-    def apply(self, res: CausalTestResult) -> bool:
-        if res.test_value.type == "ate":
-            return (0 < res.ci_low() < res.ci_high()) or (res.ci_low() < res.ci_high() < 0)
-        if res.test_value.type == "risk_ratio":
-            return (1 < res.ci_low() < res.ci_high()) or (res.ci_low() < res.ci_high() < 1)
-        raise ValueError(f"Test Value type {res.test_value.type} is not valid for this TestOutcome")
-
-    def __str__(self):
-        return "Changed"
-
-
-class NoEffect(CausalTestOutcome):
-    """An extension of TestOutcome representing that the expected causal effect should be zero."""
-
-    def apply(self, res: CausalTestResult) -> bool:
-        if res.test_value.type == "ate":
-            return (res.ci_low() < 0 < res.ci_high()) or (abs(res.test_value.value) < 1e-10)
-        if res.test_value.type == "risk_ratio":
-            return (res.ci_low() < 1 < res.ci_high()) or np.isclose(res.test_value.value, 1.0, atol=1e-10)
-        raise ValueError(f"Test Value type {res.test_value.type} is not valid for this TestOutcome")
-
-    def __str__(self):
-        return "Unchanged"

causal_testing/testing/causal_test_result.py

@@ -85,6 +85,10 @@ def ci_high(self):
             return max(self.confidence_intervals)
         return None
 
+    def ci_valid(self) -> bool:
+        """Return whether or not the result has valid confidence invervals"""
+        return self.ci_low() and self.ci_high()
+
     def summary(self):
         """Summarise the causal test result as an intuitive sentence."""
         print(

tests/testing_tests/test_causal_test_outcome.py

@@ -1,5 +1,5 @@
 import unittest
-from causal_testing.testing.causal_test_outcome import ExactValue, SomeEffect, Positive, Negative
+from causal_testing.testing.causal_test_outcome import ExactValue, SomeEffect, Positive, Negative, NoEffect
 from causal_testing.testing.causal_test_result import CausalTestResult, TestValue
 from causal_testing.testing.estimators import LinearRegressionEstimator
 from causal_testing.testing.validation import CausalValidator
@@ -86,6 +86,17 @@ def test_Positive_fail(self):
         ev = Positive()
         self.assertFalse(ev.apply(ctr))
 
+    def test_Positive_fail_ci(self):
+        test_value = TestValue(type="ate", value=0)
+        ctr = CausalTestResult(
+            estimator=self.estimator,
+            test_value=test_value,
+            confidence_intervals=[-1, 1],
+            effect_modifier_configuration=None,
+        )
+        ev = Positive()
+        self.assertFalse(ev.apply(ctr))
+
     def test_Negative_pass(self):
         test_value = TestValue(type="ate", value=-5.05)
         ctr = CausalTestResult(
@@ -108,6 +119,17 @@ def test_Negative_fail(self):
         ev = Negative()
         self.assertFalse(ev.apply(ctr))
 
+    def test_Negative_fail_ci(self):
+        test_value = TestValue(type="ate", value=0)
+        ctr = CausalTestResult(
+            estimator=self.estimator,
+            test_value=test_value,
+            confidence_intervals=[-1, 1],
+            effect_modifier_configuration=None,
+        )
+        ev = Negative()
+        self.assertFalse(ev.apply(ctr))
+
     def test_exactValue_pass(self):
         test_value = TestValue(type="ate", value=5.05)
         ctr = CausalTestResult(
@@ -119,6 +141,17 @@ def test_exactValue_pass(self):
         ev = ExactValue(5, 0.1)
         self.assertTrue(ev.apply(ctr))
 
+    def test_exactValue_pass_ci(self):
+        test_value = TestValue(type="ate", value=5.05)
+        ctr = CausalTestResult(
+            estimator=self.estimator,
+            test_value=test_value,
+            confidence_intervals=[4, 6],
+            effect_modifier_configuration=None,
+        )
+        ev = ExactValue(5, 0.1)
+        self.assertTrue(ev.apply(ctr))
+
     def test_exactValue_fail(self):
         test_value = TestValue(type="ate", value=0)
         ctr = CausalTestResult(
@@ -130,18 +163,52 @@ def test_exactValue_fail(self):
         ev = ExactValue(5, 0.1)
         self.assertFalse(ev.apply(ctr))
 
-    def test_someEffect_pass(self):
-        test_value = TestValue(type="ate", value=5.05)
+    def test_someEffect_invalid(self):
+        test_value = TestValue(type="invalid", value=5.05)
         ctr = CausalTestResult(
             estimator=self.estimator,
             test_value=test_value,
             confidence_intervals=[4.8, 6.7],
             effect_modifier_configuration=None,
         )
         ev = SomeEffect()
-        self.assertTrue(ev.apply(ctr))
+        with self.assertRaises(ValueError):
+            ev.apply(ctr)
+
+    def test_someEffect_pass_ate(self):
+        test_value = TestValue(type="ate", value=5.05)
+        ctr = CausalTestResult(
+            estimator=self.estimator,
+            test_value=test_value,
+            confidence_intervals=[4.8, 6.7],
+            effect_modifier_configuration=None,
+        )
+        self.assertTrue(SomeEffect().apply(ctr))
+        self.assertFalse(NoEffect().apply(ctr))
+
+    def test_someEffect_pass_rr(self):
+        test_value = TestValue(type="risk_ratio", value=5.05)
+        ctr = CausalTestResult(
+            estimator=self.estimator,
+            test_value=test_value,
+            confidence_intervals=[4.8, 6.7],
+            effect_modifier_configuration=None,
+        )
+        self.assertTrue(SomeEffect().apply(ctr))
+        self.assertFalse(NoEffect().apply(ctr))
 
     def test_someEffect_fail(self):
+        test_value = TestValue(type="ate", value=0)
+        ctr = CausalTestResult(
+            estimator=self.estimator,
+            test_value=test_value,
+            confidence_intervals=[-0.1, 0.2],
+            effect_modifier_configuration=None,
+        )
+        self.assertFalse(SomeEffect().apply(ctr))
+        self.assertTrue(NoEffect().apply(ctr))
+
+    def test_someEffect_str(self):
         test_value = TestValue(type="ate", value=0)
         ctr = CausalTestResult(
             estimator=self.estimator,
@@ -150,20 +217,29 @@ def test_someEffect_fail(self):
             effect_modifier_configuration=None,
         )
         ev = SomeEffect()
-        self.assertFalse(ev.apply(ctr))
         self.assertEqual(
-            str(ctr),
-            (
-                "Causal Test Result\n==============\n"
-                "Treatment: A\n"
-                "Control value: 0\n"
-                "Treatment value: 1\n"
-                "Outcome: A\n"
-                "Adjustment set: set()\n"
-                "ate: 0\n"
-                "Confidence intervals: [-0.1, 0.2]\n"
-            ),
+            ctr.to_dict(),
+            {
+                "treatment": "A",
+                "control_value": 0,
+                "treatment_value": 1,
+                "outcome": "A",
+                "adjustment_set": set(),
+                "test_value": test_value,
+                "ci_low": -0.1,
+                "ci_high": 0.2,
+            },
+        )
+
+    def test_someEffect_dict(self):
+        test_value = TestValue(type="ate", value=0)
+        ctr = CausalTestResult(
+            estimator=self.estimator,
+            test_value=test_value,
+            confidence_intervals=[-0.1, 0.2],
+            effect_modifier_configuration=None,
         )
+        ev = SomeEffect()
         self.assertEqual(
             ctr.to_dict(),
             {