diff --git a/causal_testing/main.py b/causal_testing/main.py index 9685e723..89ae0e44 100644 --- a/causal_testing/main.py +++ b/causal_testing/main.py @@ -17,7 +17,7 @@ from causal_testing.specification.causal_specification import CausalSpecification from causal_testing.testing.causal_test_case import CausalTestCase from causal_testing.testing.base_test_case import BaseTestCase -from causal_testing.testing.causal_test_outcome import NoEffect, SomeEffect, Positive, Negative +from causal_testing.testing.causal_effect import NoEffect, SomeEffect, Positive, Negative from causal_testing.testing.causal_test_result import CausalTestResult, TestValue from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator from causal_testing.estimation.logistic_regression_estimator import LogisticRegressionEstimator diff --git a/causal_testing/testing/causal_test_outcome.py b/causal_testing/testing/causal_effect.py similarity index 63% rename from causal_testing/testing/causal_test_outcome.py rename to causal_testing/testing/causal_effect.py index 6c1ea86e..a9e36d6b 100644 --- a/causal_testing/testing/causal_test_outcome.py +++ b/causal_testing/testing/causal_effect.py @@ -1,5 +1,5 @@ # pylint: disable=too-few-public-methods -"""This module contains the CausalTestOutcome abstract class, as well as the concrete extension classes: +"""This module contains the CausalEffect abstract class, as well as the concrete extension classes: ExactValue, Positive, Negative, SomeEffect, NoEffect""" from abc import ABC, abstractmethod @@ -9,7 +9,7 @@ from causal_testing.testing.causal_test_result import CausalTestResult -class CausalTestOutcome(ABC): +class CausalEffect(ABC): """An abstract class representing an expected causal effect.""" @abstractmethod @@ -23,8 +23,8 @@ def __str__(self) -> str: return type(self).__name__ -class SomeEffect(CausalTestOutcome): - """An extension of TestOutcome representing that the expected causal effect should not be zero.""" +class SomeEffect(CausalEffect): + """An extension of CausalEffect representing that the expected causal effect should not be zero.""" def apply(self, res: CausalTestResult) -> bool: if res.ci_low() is None or res.ci_high() is None: @@ -38,11 +38,11 @@ def apply(self, res: CausalTestResult) -> bool: 0 < ci_low < ci_high or ci_low < ci_high < 0 for ci_low, ci_high in zip(res.ci_low(), res.ci_high()) ) - raise ValueError(f"Test Value type {res.test_value.type} is not valid for this TestOutcome") + raise ValueError(f"Test Value type {res.test_value.type} is not valid for this CausalEffect") -class NoEffect(CausalTestOutcome): - """An extension of TestOutcome representing that the expected causal effect should be zero.""" +class NoEffect(CausalEffect): + """An extension of CausalEffect representing that the expected causal effect should be zero.""" def __init__(self, atol: float = 1e-10, ctol: float = 0.05): """ @@ -70,53 +70,64 @@ def apply(self, res: CausalTestResult) -> bool: < self.ctol ) - raise ValueError(f"Test Value type {res.test_value.type} is not valid for this TestOutcome") + raise ValueError(f"Test Value type {res.test_value.type} is not valid for this CausalEffect") -class ExactValue(SomeEffect): - """An extension of TestOutcome representing that the expected causal effect should be a specific value.""" +class ExactValue(CausalEffect): + """An extension of CausalEffect representing that the expected causal effect should be a specific value.""" + + def __init__(self, value: float, atol: float = None, ci_low: float = None, ci_high: float = None): + if (ci_low is not None) ^ (ci_high is not None): + raise ValueError("If specifying confidence intervals, must specify `ci_low` and `ci_high` parameters.") + if atol is not None and atol < 0: + raise ValueError("Tolerance must be an absolute (positive) value.") - def __init__(self, value: float, atol: float = None): self.value = value - if atol is None: - self.atol = abs(value * 0.05) - else: - self.atol = atol - if self.atol < 0: - raise ValueError("Tolerance must be an absolute value.") + self.ci_low = ci_low + self.ci_high = ci_high + self.atol = atol if atol is not None else abs(value * 0.05) + + if self.ci_low is not None and self.ci_high is not None: + if not self.ci_low <= self.value <= self.ci_high: + raise ValueError("Specified value falls outside the specified confidence intervals.") + if self.value - self.atol < self.ci_low or self.value + self.atol > self.ci_high: + raise ValueError( + "Arithmetic tolerance falls outside the confidence intervals." + "Try specifying a smaller value of atol." + ) def apply(self, res: CausalTestResult) -> bool: - if res.ci_valid(): - return super().apply(res) and np.isclose(res.test_value.value, self.value, atol=self.atol) - return np.isclose(res.test_value.value, self.value, atol=self.atol) + close = np.isclose(res.test_value.value, self.value, atol=self.atol) + if res.ci_valid() and self.ci_low is not None and self.ci_high is not None: + return all( + close and self.ci_low <= ci_low and self.ci_high >= ci_high + for ci_low, ci_high in zip(res.ci_low(), res.ci_high()) + ) + return close def __str__(self): return f"ExactValue: {self.value}±{self.atol}" class Positive(SomeEffect): - """An extension of TestOutcome representing that the expected causal effect should be positive. + """An extension of CausalEffect representing that the expected causal effect should be positive. Currently only single values are supported for the test value""" def apply(self, res: CausalTestResult) -> bool: - if res.ci_valid() and not super().apply(res): - return False if len(res.test_value.value) > 1: raise ValueError("Positive Effects are currently only supported on single float datatypes") if res.test_value.type in {"ate", "coefficient"}: return bool(res.test_value.value[0] > 0) if res.test_value.type == "risk_ratio": return bool(res.test_value.value[0] > 1) - raise ValueError(f"Test Value type {res.test_value.type} is not valid for this TestOutcome") + raise ValueError(f"Test Value type {res.test_value.type} is not valid for this CausalEffect") class Negative(SomeEffect): - """An extension of TestOutcome representing that the expected causal effect should be negative. + """An extension of CausalEffect representing that the expected causal effect should be negative. Currently only single values are supported for the test value""" def apply(self, res: CausalTestResult) -> bool: - if res.ci_valid() and not super().apply(res): - return False if len(res.test_value.value) > 1: raise ValueError("Negative Effects are currently only supported on single float datatypes") if res.test_value.type in {"ate", "coefficient"}: @@ -124,4 +135,4 @@ def apply(self, res: CausalTestResult) -> bool: if res.test_value.type == "risk_ratio": return bool(res.test_value.value[0] < 1) # Dead code but necessary for pylint - raise ValueError(f"Test Value type {res.test_value.type} is not valid for this TestOutcome") + raise ValueError(f"Test Value type {res.test_value.type} is not valid for this CausalEffect") diff --git a/causal_testing/testing/causal_test_case.py b/causal_testing/testing/causal_test_case.py index 30343560..64fb75cb 100644 --- a/causal_testing/testing/causal_test_case.py +++ b/causal_testing/testing/causal_test_case.py @@ -4,7 +4,7 @@ from typing import Any from causal_testing.specification.variable import Variable -from causal_testing.testing.causal_test_outcome import CausalTestOutcome +from causal_testing.testing.causal_effect import CausalEffect from causal_testing.testing.base_test_case import BaseTestCase from causal_testing.estimation.abstract_estimator import Estimator from causal_testing.testing.causal_test_result import CausalTestResult, TestValue @@ -26,7 +26,7 @@ def __init__( # pylint: disable=too-many-arguments self, base_test_case: BaseTestCase, - expected_causal_effect: CausalTestOutcome, + expected_causal_effect: CausalEffect, estimate_type: str = "ate", estimate_params: dict = None, effect_modifier_configuration: dict[Variable:Any] = None, diff --git a/dafni/main_dafni.py b/dafni/main_dafni.py index ed310cda..a3d8f5c6 100644 --- a/dafni/main_dafni.py +++ b/dafni/main_dafni.py @@ -10,7 +10,7 @@ import pandas as pd from causal_testing.specification.scenario import Scenario from causal_testing.specification.variable import Input, Output -from causal_testing.testing.causal_test_outcome import Positive, Negative, NoEffect, SomeEffect +from causal_testing.testing.causal_effect import Positive, Negative, NoEffect, SomeEffect from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator from causal_testing.estimation.logistic_regression_estimator import LogisticRegressionEstimator from causal_testing.json_front.json_class import JsonUtility diff --git a/docs/source/usage.rst b/docs/source/usage.rst index adab3719..d4be317a 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -44,7 +44,7 @@ the given output and input and the desired effect. This information is the minim from causal_testing.testing.base_test_case import BaseTestCase from causal_testing.testing.causal_test_case import CausalTestCase - from causal_testing.testing.causal_test_outcome import Positive + from causal_testing.testing.causal_effect import Positive from causal_testing.testing.effect import Effect base_test_case = BaseTestCase( diff --git a/examples/covasim_/doubling_beta/example_beta.py b/examples/covasim_/doubling_beta/example_beta.py index 1655b352..e1a7be33 100644 --- a/examples/covasim_/doubling_beta/example_beta.py +++ b/examples/covasim_/doubling_beta/example_beta.py @@ -7,7 +7,7 @@ import numpy as np from causal_testing.specification.variable import Input, Output from causal_testing.testing.causal_test_case import CausalTestCase -from causal_testing.testing.causal_test_outcome import Positive +from causal_testing.testing.causal_effect import Positive from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator from causal_testing.testing.base_test_case import BaseTestCase diff --git a/examples/covasim_/vaccinating_elderly/example_vaccine.py b/examples/covasim_/vaccinating_elderly/example_vaccine.py index 50936a0a..dede5d9e 100644 --- a/examples/covasim_/vaccinating_elderly/example_vaccine.py +++ b/examples/covasim_/vaccinating_elderly/example_vaccine.py @@ -7,7 +7,7 @@ from causal_testing.specification.variable import Input, Output from causal_testing.specification.causal_specification import CausalSpecification from causal_testing.testing.causal_test_case import CausalTestCase -from causal_testing.testing.causal_test_outcome import Positive, Negative, NoEffect +from causal_testing.testing.causal_effect import Positive, Negative, NoEffect from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator from causal_testing.testing.base_test_case import BaseTestCase diff --git a/examples/lr91/example_max_conductances.py b/examples/lr91/example_max_conductances.py index 6bd486c2..50ca0728 100644 --- a/examples/lr91/example_max_conductances.py +++ b/examples/lr91/example_max_conductances.py @@ -6,7 +6,7 @@ from causal_testing.specification.variable import Input, Output from causal_testing.specification.causal_specification import CausalSpecification from causal_testing.testing.causal_test_case import CausalTestCase -from causal_testing.testing.causal_test_outcome import Positive, Negative, NoEffect +from causal_testing.testing.causal_effect import Positive, Negative, NoEffect from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator from causal_testing.testing.base_test_case import BaseTestCase from matplotlib.pyplot import rcParams diff --git a/examples/poisson-line-process/example_pure_python.py b/examples/poisson-line-process/example_pure_python.py index 0788896c..04f18616 100644 --- a/examples/poisson-line-process/example_pure_python.py +++ b/examples/poisson-line-process/example_pure_python.py @@ -8,7 +8,7 @@ from causal_testing.specification.variable import Input, Output from causal_testing.specification.causal_specification import CausalSpecification from causal_testing.testing.causal_test_case import CausalTestCase -from causal_testing.testing.causal_test_outcome import ExactValue, Positive +from causal_testing.testing.causal_effect import ExactValue, Positive from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator from causal_testing.estimation.abstract_estimator import Estimator from causal_testing.testing.base_test_case import BaseTestCase diff --git a/tests/testing_tests/test_causal_test_outcome.py b/tests/testing_tests/test_causal_effect.py similarity index 85% rename from tests/testing_tests/test_causal_test_outcome.py rename to tests/testing_tests/test_causal_effect.py index e1a7c40a..89d02ae1 100644 --- a/tests/testing_tests/test_causal_test_outcome.py +++ b/tests/testing_tests/test_causal_effect.py @@ -1,6 +1,6 @@ import unittest import pandas as pd -from causal_testing.testing.causal_test_outcome import ExactValue, SomeEffect, Positive, Negative, NoEffect +from causal_testing.testing.causal_effect import ExactValue, SomeEffect, Positive, Negative, NoEffect from causal_testing.testing.causal_test_result import CausalTestResult, TestValue from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator from causal_testing.utils.validation import CausalValidator @@ -8,8 +8,8 @@ from causal_testing.specification.variable import Input, Output -class TestCausalTestOutcome(unittest.TestCase): - """Test the TestCausalTestOutcome basic methods.""" +class TestCausalEffect(unittest.TestCase): + """Test the TestCausalEffect basic methods.""" def setUp(self) -> None: base_test_case = BaseTestCase(Input("A", float), Output("A", float)) @@ -181,6 +181,28 @@ def test_exactValue_pass_ci(self): ev = ExactValue(5, 0.1) self.assertTrue(ev.apply(ctr)) + def test_exactValue_ci_pass_ci(self): + test_value = TestValue(type="ate", value=pd.Series(5.05)) + ctr = CausalTestResult( + estimator=self.estimator, + test_value=test_value, + confidence_intervals=[pd.Series(4.1), pd.Series(5.9)], + effect_modifier_configuration=None, + ) + ev = ExactValue(5, ci_low=4, ci_high=6) + self.assertTrue(ev.apply(ctr)) + + def test_exactValue_ci_fail_ci(self): + test_value = TestValue(type="ate", value=pd.Series(5.05)) + ctr = CausalTestResult( + estimator=self.estimator, + test_value=test_value, + confidence_intervals=[pd.Series(3.9), pd.Series(6.1)], + effect_modifier_configuration=None, + ) + ev = ExactValue(5, ci_low=4, ci_high=6) + self.assertFalse(ev.apply(ctr)) + def test_exactValue_fail(self): test_value = TestValue(type="ate", value=pd.Series(0)) ctr = CausalTestResult( @@ -196,6 +218,22 @@ def test_invalid_atol(self): with self.assertRaises(ValueError): ExactValue(5, -0.1) + def test_unspecified_ci_high(self): + with self.assertRaises(ValueError): + ExactValue(5, ci_low=-0.1) + + def test_unspecified_ci_low(self): + with self.assertRaises(ValueError): + ExactValue(5, ci_high=-0.1) + + def test_invalid_ci_range(self): + with self.assertRaises(ValueError): + ExactValue(5, ci_low=6, ci_high=7, atol=0.05) + + def test_invalid_ci_atol(self): + with self.assertRaises(ValueError): + ExactValue(1000, ci_low=999, ci_high=1001, atol=50) + def test_invalid(self): test_value = TestValue(type="invalid", value=pd.Series(5.05)) ctr = CausalTestResult( @@ -257,6 +295,16 @@ def test_someEffect_fail(self): self.assertFalse(SomeEffect().apply(ctr)) self.assertTrue(NoEffect().apply(ctr)) + def test_someEffect_None(self): + test_value = TestValue(type="ate", value=pd.Series(0)) + ctr = CausalTestResult( + estimator=self.estimator, + test_value=test_value, + confidence_intervals=None, + effect_modifier_configuration=None, + ) + self.assertEqual(SomeEffect().apply(ctr), None) + def test_someEffect_str(self): test_value = TestValue(type="ate", value=0) ctr = CausalTestResult( diff --git a/tests/testing_tests/test_causal_test_adequacy.py b/tests/testing_tests/test_causal_test_adequacy.py index 56149024..aae1fc90 100644 --- a/tests/testing_tests/test_causal_test_adequacy.py +++ b/tests/testing_tests/test_causal_test_adequacy.py @@ -9,7 +9,7 @@ from causal_testing.testing.base_test_case import BaseTestCase from causal_testing.testing.causal_test_case import CausalTestCase from causal_testing.testing.causal_test_adequacy import DAGAdequacy -from causal_testing.testing.causal_test_outcome import NoEffect, SomeEffect +from causal_testing.testing.causal_effect import NoEffect, SomeEffect from causal_testing.specification.scenario import Scenario from causal_testing.testing.causal_test_adequacy import DataAdequacy from causal_testing.specification.variable import Input, Output diff --git a/tests/testing_tests/test_causal_test_case.py b/tests/testing_tests/test_causal_test_case.py index 17390819..4c4a69c6 100644 --- a/tests/testing_tests/test_causal_test_case.py +++ b/tests/testing_tests/test_causal_test_case.py @@ -9,7 +9,7 @@ from causal_testing.specification.variable import Input, Output from causal_testing.specification.causal_dag import CausalDAG from causal_testing.testing.causal_test_case import CausalTestCase -from causal_testing.testing.causal_test_outcome import ExactValue +from causal_testing.testing.causal_effect import ExactValue from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator from causal_testing.testing.base_test_case import BaseTestCase