Bootstrap kurtosis coverage

Author: jmafoster1

causal_testing/json_front/json_class.py

@@ -11,6 +11,7 @@
 from statistics import StatisticsError
 
 import pandas as pd
+import numpy as np
 import scipy
 from fitter import Fitter, get_common_distributions
 
@@ -269,23 +270,10 @@ def _execute_test_case(
 
         if "coverage" in test and test["coverage"]:
             adequacy = DataAdequacy(causal_test_case, causal_test_engine, estimation_model)
-            results = adequacy.measure_adequacy_bootstrap(100)
-            outcomes = [causal_test_case.expected_causal_effect.apply(c) for c in results]
-            coverage = pd.DataFrame(c.to_dict() for c in results)[["effect_estimate", "ci_low", "ci_high"]]
-            coverage["pass"] = outcomes
-            std = coverage.std(numeric_only=True)
-            self._append_to_file(f"COVERAGE: {coverage['pass'].sum()}", logging.INFO)
-            # std["pass"] = coverage["pass"].sum()
-            # print(coverage)
-            # print(std)
-
-            # k_folds = adequacy.measure_adequacy_k_folds()
-
-            # import matplotlib.pyplot as plt
-            #
-            # plt.hist(coverage["ci_low"], alpha=0.8)
-            # plt.hist(coverage["ci_high"], alpha=0.8)
-            # plt.show()
+            effect_estimate, ci_low, ci_high, outcomes = adequacy.measure_adequacy(100)
+
+            self._append_to_file(f"KURTOSIS: {effect_estimate.mean()}", logging.INFO)
+            self._append_to_file(f"PASSING:\n{sum(outcomes)}/{len(outcomes)}", logging.INFO)
 
         if causal_test_result.ci_low() is not None and causal_test_result.ci_high() is not None:
             result_string = (
@@ -398,7 +386,6 @@ def get_args(test_args=None) -> argparse.Namespace:
         parser.add_argument(
             "--log_path",
             help="Specify a directory to change the location of the log file",
-            default="./json_frontend.log",
         )
         parser.add_argument(
             "--data_path",

causal_testing/testing/causal_test_adequacy.py

@@ -12,6 +12,7 @@
 from sklearn.metrics import mean_squared_error as mse
 import numpy as np
 from sklearn.model_selection import cross_val_score
+import pandas as pd
 
 
 class DAGAdequacy:
@@ -38,21 +39,30 @@ def __init__(self, test_case: CausalTestCase, test_engine: CausalTestEngine, est
         self.test_engine = test_engine
         self.estimator = estimator
 
-    def measure_adequacy_bootstrap(self, bootstrap_size: int = 100):
+    def measure_adequacy(self, bootstrap_size: int = 100):
         results = []
         for i in range(bootstrap_size):
             estimator = deepcopy(self.estimator)
             estimator.df = estimator.df.sample(len(estimator.df), replace=True, random_state=i)
             results.append(self.test_engine.execute_test(estimator, self.test_case))
-        return results
+        outcomes = [self.test_case.expected_causal_effect.apply(c) for c in results]
+        results = pd.DataFrame(c.to_dict() for c in results)[["effect_estimate", "ci_low", "ci_high"]]
 
-    def measure_adequacy_k_folds(self, k: int = 10, random_state=0):
-        results = []
-        kf = KFold(n_splits=k, shuffle=True, random_state=random_state)
-        for train_inx, test_inx in kf.split(self.estimator.df):
-            estimator = deepcopy(self.estimator)
-            test = estimator.df.iloc[test_inx]
-            estimator.df = estimator.df.iloc[train_inx]
-            test_result = estimator.model.predict(test)
-            results.append(np.sqrt(mse(test_result, test[self.test_case.base_test_case.outcome_variable.name])).mean())
-        return np.mean(results)
+        def convert_to_df(field):
+            converted = []
+            for r in results[field]:
+                if isinstance(r, float):
+                    converted.append(
+                        pd.DataFrame({self.test_case.base_test_case.treatment_variable.name: [r]}).transpose()
+                    )
+                else:
+                    converted.append(r)
+            return converted
+
+        for field in ["effect_estimate", "ci_low", "ci_high"]:
+            results[field] = convert_to_df(field)
+
+        effect_estimate = pd.concat(results["effect_estimate"].tolist(), axis=1).transpose().reset_index(drop=True)
+        ci_low = pd.concat(results["ci_low"].tolist(), axis=1).transpose()
+        ci_high = pd.concat(results["ci_high"].tolist(), axis=1).transpose()
+        return effect_estimate.kurtosis(), ci_low.kurtosis(), ci_high.kurtosis(), outcomes