Jmafoster1/exact value confidence intervals

causal_testing/main.py

@@ -17,7 +17,7 @@
 from causal_testing.specification.causal_specification import CausalSpecification
 from causal_testing.testing.causal_test_case import CausalTestCase
 from causal_testing.testing.base_test_case import BaseTestCase
-from causal_testing.testing.causal_test_outcome import NoEffect, SomeEffect, Positive, Negative
+from causal_testing.testing.causal_effect import NoEffect, SomeEffect, Positive, Negative
 from causal_testing.testing.causal_test_result import CausalTestResult, TestValue
 from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator
 from causal_testing.estimation.logistic_regression_estimator import LogisticRegressionEstimator

causal_testing/testing/causal_test_outcome.py → causal_testing/testing/causal_effect.py

@@ -1,5 +1,5 @@
 # pylint: disable=too-few-public-methods
-"""This module contains the CausalTestOutcome abstract class, as well as the concrete extension classes:
+"""This module contains the CausalEffect abstract class, as well as the concrete extension classes:
 ExactValue, Positive, Negative, SomeEffect, NoEffect"""
 
 from abc import ABC, abstractmethod
@@ -9,7 +9,7 @@
 from causal_testing.testing.causal_test_result import CausalTestResult
 
 
-class CausalTestOutcome(ABC):
+class CausalEffect(ABC):
     """An abstract class representing an expected causal effect."""
 
     @abstractmethod
@@ -23,8 +23,8 @@ def __str__(self) -> str:
         return type(self).__name__
 
 
-class SomeEffect(CausalTestOutcome):
-    """An extension of TestOutcome representing that the expected causal effect should not be zero."""
+class SomeEffect(CausalEffect):
+    """An extension of CausalEffect representing that the expected causal effect should not be zero."""
 
     def apply(self, res: CausalTestResult) -> bool:
         if res.ci_low() is None or res.ci_high() is None:
@@ -38,11 +38,11 @@ def apply(self, res: CausalTestResult) -> bool:
                 0 < ci_low < ci_high or ci_low < ci_high < 0 for ci_low, ci_high in zip(res.ci_low(), res.ci_high())
             )
 
-        raise ValueError(f"Test Value type {res.test_value.type} is not valid for this TestOutcome")
+        raise ValueError(f"Test Value type {res.test_value.type} is not valid for this CausalEffect")
 
 
-class NoEffect(CausalTestOutcome):
-    """An extension of TestOutcome representing that the expected causal effect should be zero."""
+class NoEffect(CausalEffect):
+    """An extension of CausalEffect representing that the expected causal effect should be zero."""
 
     def __init__(self, atol: float = 1e-10, ctol: float = 0.05):
         """
@@ -70,58 +70,69 @@ def apply(self, res: CausalTestResult) -> bool:
                 < self.ctol
             )
 
-        raise ValueError(f"Test Value type {res.test_value.type} is not valid for this TestOutcome")
+        raise ValueError(f"Test Value type {res.test_value.type} is not valid for this CausalEffect")
 
 
-class ExactValue(SomeEffect):
-    """An extension of TestOutcome representing that the expected causal effect should be a specific value."""
+class ExactValue(CausalEffect):
+    """An extension of CausalEffect representing that the expected causal effect should be a specific value."""
+
+    def __init__(self, value: float, atol: float = None, ci_low: float = None, ci_high: float = None):
+        if (ci_low is not None) ^ (ci_high is not None):
+            raise ValueError("If specifying confidence intervals, must specify `ci_low` and `ci_high` parameters.")
+        if atol is not None and atol < 0:
+            raise ValueError("Tolerance must be an absolute (positive) value.")
 
-    def __init__(self, value: float, atol: float = None):
         self.value = value
-        if atol is None:
-            self.atol = abs(value * 0.05)
-        else:
-            self.atol = atol
-        if self.atol < 0:
-            raise ValueError("Tolerance must be an absolute value.")
+        self.ci_low = ci_low
+        self.ci_high = ci_high
+        self.atol = atol if atol is not None else abs(value * 0.05)
+
+        if self.ci_low is not None and self.ci_high is not None:
+            if not self.ci_low <= self.value <= self.ci_high:
+                raise ValueError("Specified value falls outside the specified confidence intervals.")
+            if self.value - self.atol < self.ci_low or self.value + self.atol > self.ci_high:
+                raise ValueError(
+                    "Arithmetic tolerance falls outside the confidence intervals."
+                    "Try specifying a smaller value of atol."
+                )
 
     def apply(self, res: CausalTestResult) -> bool:
-        if res.ci_valid():
-            return super().apply(res) and np.isclose(res.test_value.value, self.value, atol=self.atol)
-        return np.isclose(res.test_value.value, self.value, atol=self.atol)
+        close = np.isclose(res.test_value.value, self.value, atol=self.atol)
+        if res.ci_valid() and self.ci_low is not None and self.ci_high is not None:
+            return all(
+                close and self.ci_low <= ci_low and self.ci_high >= ci_high
+                for ci_low, ci_high in zip(res.ci_low(), res.ci_high())
+            )
+        return close
 
     def __str__(self):
         return f"ExactValue: {self.value}±{self.atol}"
 
 
 class Positive(SomeEffect):
-    """An extension of TestOutcome representing that the expected causal effect should be positive.
+    """An extension of CausalEffect representing that the expected causal effect should be positive.
     Currently only single values are supported for the test value"""
 
     def apply(self, res: CausalTestResult) -> bool:
-        if res.ci_valid() and not super().apply(res):
-            return False
         if len(res.test_value.value) > 1:
             raise ValueError("Positive Effects are currently only supported on single float datatypes")
         if res.test_value.type in {"ate", "coefficient"}:
             return bool(res.test_value.value[0] > 0)
         if res.test_value.type == "risk_ratio":
             return bool(res.test_value.value[0] > 1)
-        raise ValueError(f"Test Value type {res.test_value.type} is not valid for this TestOutcome")
+        raise ValueError(f"Test Value type {res.test_value.type} is not valid for this CausalEffect")
 
 
 class Negative(SomeEffect):
-    """An extension of TestOutcome representing that the expected causal effect should be negative.
+    """An extension of CausalEffect representing that the expected causal effect should be negative.
     Currently only single values are supported for the test value"""
 
     def apply(self, res: CausalTestResult) -> bool:
-        if res.ci_valid() and not super().apply(res):
-            return False
         if len(res.test_value.value) > 1:
             raise ValueError("Negative Effects are currently only supported on single float datatypes")
         if res.test_value.type in {"ate", "coefficient"}:
             return bool(res.test_value.value[0] < 0)
         if res.test_value.type == "risk_ratio":
             return bool(res.test_value.value[0] < 1)
         # Dead code but necessary for pylint
-        raise ValueError(f"Test Value type {res.test_value.type} is not valid for this TestOutcome")
+        raise ValueError(f"Test Value type {res.test_value.type} is not valid for this CausalEffect")

causal_testing/testing/causal_test_case.py

@@ -4,7 +4,7 @@
 from typing import Any
 
 from causal_testing.specification.variable import Variable
-from causal_testing.testing.causal_test_outcome import CausalTestOutcome
+from causal_testing.testing.causal_effect import CausalEffect
 from causal_testing.testing.base_test_case import BaseTestCase
 from causal_testing.estimation.abstract_estimator import Estimator
 from causal_testing.testing.causal_test_result import CausalTestResult, TestValue
@@ -26,7 +26,7 @@ def __init__(
         # pylint: disable=too-many-arguments
         self,
         base_test_case: BaseTestCase,
-        expected_causal_effect: CausalTestOutcome,
+        expected_causal_effect: CausalEffect,
         estimate_type: str = "ate",
         estimate_params: dict = None,
         effect_modifier_configuration: dict[Variable:Any] = None,

dafni/main_dafni.py

@@ -10,7 +10,7 @@
 import pandas as pd
 from causal_testing.specification.scenario import Scenario
 from causal_testing.specification.variable import Input, Output
-from causal_testing.testing.causal_test_outcome import Positive, Negative, NoEffect, SomeEffect
+from causal_testing.testing.causal_effect import Positive, Negative, NoEffect, SomeEffect
 from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator
 from causal_testing.estimation.logistic_regression_estimator import LogisticRegressionEstimator
 from causal_testing.json_front.json_class import JsonUtility

docs/source/usage.rst

@@ -44,7 +44,7 @@ the given output and input and the desired effect. This information is the minim
 
    from causal_testing.testing.base_test_case import BaseTestCase
    from causal_testing.testing.causal_test_case import CausalTestCase
-   from causal_testing.testing.causal_test_outcome import Positive
+   from causal_testing.testing.causal_effect import Positive
    from causal_testing.testing.effect import Effect
 
    base_test_case = BaseTestCase(

examples/covasim_/doubling_beta/example_beta.py

@@ -7,7 +7,7 @@
 import numpy as np
 from causal_testing.specification.variable import Input, Output
 from causal_testing.testing.causal_test_case import CausalTestCase
-from causal_testing.testing.causal_test_outcome import Positive
+from causal_testing.testing.causal_effect import Positive
 from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator
 from causal_testing.testing.base_test_case import BaseTestCase
 

examples/covasim_/vaccinating_elderly/example_vaccine.py

@@ -7,7 +7,7 @@
 from causal_testing.specification.variable import Input, Output
 from causal_testing.specification.causal_specification import CausalSpecification
 from causal_testing.testing.causal_test_case import CausalTestCase
-from causal_testing.testing.causal_test_outcome import Positive, Negative, NoEffect
+from causal_testing.testing.causal_effect import Positive, Negative, NoEffect
 from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator
 from causal_testing.testing.base_test_case import BaseTestCase
 

examples/lr91/example_max_conductances.py

@@ -6,7 +6,7 @@
 from causal_testing.specification.variable import Input, Output
 from causal_testing.specification.causal_specification import CausalSpecification
 from causal_testing.testing.causal_test_case import CausalTestCase
-from causal_testing.testing.causal_test_outcome import Positive, Negative, NoEffect
+from causal_testing.testing.causal_effect import Positive, Negative, NoEffect
 from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator
 from causal_testing.testing.base_test_case import BaseTestCase
 from matplotlib.pyplot import rcParams

examples/poisson-line-process/example_pure_python.py

@@ -8,7 +8,7 @@
 from causal_testing.specification.variable import Input, Output
 from causal_testing.specification.causal_specification import CausalSpecification
 from causal_testing.testing.causal_test_case import CausalTestCase
-from causal_testing.testing.causal_test_outcome import ExactValue, Positive
+from causal_testing.testing.causal_effect import ExactValue, Positive
 from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator
 from causal_testing.estimation.abstract_estimator import Estimator
 from causal_testing.testing.base_test_case import BaseTestCase

tests/testing_tests/test_causal_test_outcome.py → tests/testing_tests/test_causal_effect.py

@@ -1,15 +1,15 @@
 import unittest
 import pandas as pd
-from causal_testing.testing.causal_test_outcome import ExactValue, SomeEffect, Positive, Negative, NoEffect
+from causal_testing.testing.causal_effect import ExactValue, SomeEffect, Positive, Negative, NoEffect
 from causal_testing.testing.causal_test_result import CausalTestResult, TestValue
 from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator
 from causal_testing.utils.validation import CausalValidator
 from causal_testing.testing.base_test_case import BaseTestCase
 from causal_testing.specification.variable import Input, Output
 
 
-class TestCausalTestOutcome(unittest.TestCase):
-    """Test the TestCausalTestOutcome basic methods."""
+class TestCausalEffect(unittest.TestCase):
+    """Test the TestCausalEffect basic methods."""
 
     def setUp(self) -> None:
         base_test_case = BaseTestCase(Input("A", float), Output("A", float))
@@ -181,6 +181,28 @@ def test_exactValue_pass_ci(self):
         ev = ExactValue(5, 0.1)
         self.assertTrue(ev.apply(ctr))
 
+    def test_exactValue_ci_pass_ci(self):
+        test_value = TestValue(type="ate", value=pd.Series(5.05))
+        ctr = CausalTestResult(
+            estimator=self.estimator,
+            test_value=test_value,
+            confidence_intervals=[pd.Series(4.1), pd.Series(5.9)],
+            effect_modifier_configuration=None,
+        )
+        ev = ExactValue(5, ci_low=4, ci_high=6)
+        self.assertTrue(ev.apply(ctr))
+
+    def test_exactValue_ci_fail_ci(self):
+        test_value = TestValue(type="ate", value=pd.Series(5.05))
+        ctr = CausalTestResult(
+            estimator=self.estimator,
+            test_value=test_value,
+            confidence_intervals=[pd.Series(3.9), pd.Series(6.1)],
+            effect_modifier_configuration=None,
+        )
+        ev = ExactValue(5, ci_low=4, ci_high=6)
+        self.assertFalse(ev.apply(ctr))
+
     def test_exactValue_fail(self):
         test_value = TestValue(type="ate", value=pd.Series(0))
         ctr = CausalTestResult(
@@ -196,6 +218,22 @@ def test_invalid_atol(self):
         with self.assertRaises(ValueError):
             ExactValue(5, -0.1)
 
+    def test_unspecified_ci_high(self):
+        with self.assertRaises(ValueError):
+            ExactValue(5, ci_low=-0.1)
+
+    def test_unspecified_ci_low(self):
+        with self.assertRaises(ValueError):
+            ExactValue(5, ci_high=-0.1)
+
+    def test_invalid_ci_range(self):
+        with self.assertRaises(ValueError):
+            ExactValue(5, ci_low=6, ci_high=7, atol=0.05)
+
+    def test_invalid_ci_atol(self):
+        with self.assertRaises(ValueError):
+            ExactValue(1000, ci_low=999, ci_high=1001, atol=50)
+
     def test_invalid(self):
         test_value = TestValue(type="invalid", value=pd.Series(5.05))
         ctr = CausalTestResult(
@@ -257,6 +295,16 @@ def test_someEffect_fail(self):
         self.assertFalse(SomeEffect().apply(ctr))
         self.assertTrue(NoEffect().apply(ctr))
 
+    def test_someEffect_None(self):
+        test_value = TestValue(type="ate", value=pd.Series(0))
+        ctr = CausalTestResult(
+            estimator=self.estimator,
+            test_value=test_value,
+            confidence_intervals=None,
+            effect_modifier_configuration=None,
+        )
+        self.assertEqual(SomeEffect().apply(ctr), None)
+
     def test_someEffect_str(self):
         test_value = TestValue(type="ate", value=0)
         ctr = CausalTestResult(

tests/testing_tests/test_causal_test_adequacy.py

@@ -9,7 +9,7 @@
 from causal_testing.testing.base_test_case import BaseTestCase
 from causal_testing.testing.causal_test_case import CausalTestCase
 from causal_testing.testing.causal_test_adequacy import DAGAdequacy
-from causal_testing.testing.causal_test_outcome import NoEffect, SomeEffect
+from causal_testing.testing.causal_effect import NoEffect, SomeEffect
 from causal_testing.specification.scenario import Scenario
 from causal_testing.testing.causal_test_adequacy import DataAdequacy
 from causal_testing.specification.variable import Input, Output

tests/testing_tests/test_causal_test_case.py

@@ -9,7 +9,7 @@
 from causal_testing.specification.variable import Input, Output
 from causal_testing.specification.causal_dag import CausalDAG
 from causal_testing.testing.causal_test_case import CausalTestCase
-from causal_testing.testing.causal_test_outcome import ExactValue
+from causal_testing.testing.causal_effect import ExactValue
 from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator
 from causal_testing.testing.base_test_case import BaseTestCase