Merge pull request #184 from CITCOM-project/json_fitter

Logging causal test results

Author: jmafoster1

causal_testing/json_front/json_class.py

@@ -21,6 +21,7 @@
 from causal_testing.specification.scenario import Scenario
 from causal_testing.specification.variable import Input, Meta, Output
 from causal_testing.testing.causal_test_case import CausalTestCase
+from causal_testing.testing.causal_test_result import CausalTestResult
 from causal_testing.testing.causal_test_engine import CausalTestEngine
 from causal_testing.testing.estimators import Estimator
 from causal_testing.testing.base_test_case import BaseTestCase
@@ -124,28 +125,29 @@ def run_json_tests(self, effects: dict, estimators: dict, f_flag: bool = False,
                         effect=test.get("effect", "direct"),
                     )
                     assert len(test["expected_effect"]) == 1, "Can only have one expected effect."
-                    concrete_tests = [
-                        CausalTestCase(
-                            base_test_case=base_test_case,
-                            expected_causal_effect=next(
-                                effects[effect] for variable, effect in test["expected_effect"].items()
-                            ),
-                            estimate_type="coefficient",
-                            effect_modifier_configuration={
-                                self.scenario.variables[v] for v in test.get("effect_modifiers", [])
-                            },
-                        )
-                    ]
-                    failures = self._execute_tests(concrete_tests, test, f_flag)
+                    causal_test_case = CausalTestCase(
+                        base_test_case=base_test_case,
+                        expected_causal_effect=next(
+                            effects[effect] for variable, effect in test["expected_effect"].items()
+                        ),
+                        estimate_type="coefficient",
+                        effect_modifier_configuration={
+                            self.scenario.variables[v] for v in test.get("effect_modifiers", [])
+                        },
+                    )
+                    result = self._execute_test_case(causal_test_case=causal_test_case, test=test, f_flag=f_flag)
                     msg = (
                         f"Executing test: {test['name']} \n"
-                        + f"  {concrete_tests[0]} \n"
-                        + f"  {failures}/{len(concrete_tests)} failed for {test['name']}"
+                        + f"  {causal_test_case} \n"
+                        + "  "
+                        + ("\n  ").join(str(result[1]).split("\n"))
+                        + "==============\n"
+                        + f"  Result: {'FAILED' if result[0] else 'Passed'}"
                     )
                 else:
                     abstract_test = self._create_abstract_test_case(test, mutates, effects)
-                    concrete_tests, dummy = abstract_test.generate_concrete_tests(5, 0.05)
-                    failures = self._execute_tests(concrete_tests, test, f_flag)
+                    concrete_tests, _ = abstract_test.generate_concrete_tests(5, 0.05)
+                    failures, _ = self._execute_tests(concrete_tests, test, f_flag)
 
                     msg = (
                         f"Executing test: {test['name']} \n"
@@ -173,29 +175,28 @@ def run_json_tests(self, effects: dict, estimators: dict, f_flag: bool = False,
                     treatment_value=test["treatment_value"],
                     estimate_type=test["estimate_type"],
                 )
-                if self._execute_test_case(causal_test_case=causal_test_case, test=test, f_flag=f_flag):
-                    result = "failed"
-                else:
-                    result = "passed"
+                failed, _ = self._execute_test_case(causal_test_case=causal_test_case, test=test, f_flag=f_flag)
 
                 msg = (
                     f"Executing concrete test: {test['name']} \n"
                     + f"treatment variable: {test['treatment_variable']} \n"
                     + f"outcome_variable = {outcome_variable} \n"
                     + f"control value = {test['control_value']}, treatment value = {test['treatment_value']} \n"
-                    + f"result - {result}"
+                    + f"Result: {'FAILED' if failed else 'Passed'}"
                 )
                 self._append_to_file(msg, logging.INFO)
 
     def _execute_tests(self, concrete_tests, test, f_flag):
         failures = 0
+        details = []
         if "formula" in test:
             self._append_to_file(f"Estimator formula used for test: {test['formula']}")
         for concrete_test in concrete_tests:
-            failed = self._execute_test_case(concrete_test, test, f_flag)
+            failed, result = self._execute_test_case(concrete_test, test, f_flag)
+            details.append(result)
             if failed:
                 failures += 1
-        return failures
+        return failures, details
 
     def _json_parse(self):
         """Parse a JSON input file into inputs, outputs, metas and a test plan"""
@@ -213,7 +214,9 @@ def _populate_metas(self):
         for meta in self.scenario.variables_of_type(Meta):
             meta.populate(self.data)
 
-    def _execute_test_case(self, causal_test_case: CausalTestCase, test: Iterable[Mapping], f_flag: bool) -> bool:
+    def _execute_test_case(
+        self, causal_test_case: CausalTestCase, test: Iterable[Mapping], f_flag: bool
+    ) -> (bool, CausalTestResult):
         """Executes a singular test case, prints the results and returns the test case result
         :param causal_test_case: The concrete test case to be executed
         :param test: Single JSON test definition stored in a mapping (dict)
@@ -249,7 +252,7 @@ def _execute_test_case(self, causal_test_case: CausalTestCase, test: Iterable[Ma
                 )
             failed = True
             logger.warning("   FAILED- expected %s, got %s", causal_test_case.expected_causal_effect, result_string)
-        return failed
+        return failed, causal_test_result
 
     def _setup_test(
         self, causal_test_case: CausalTestCase, test: Mapping, conditions: list[str] = None

causal_testing/testing/causal_test_result.py

@@ -3,6 +3,7 @@
 """
 from typing import Any
 from dataclasses import dataclass
+import pandas as pd
 
 from causal_testing.testing.estimators import Estimator
 from causal_testing.specification.variable import Variable
@@ -87,7 +88,7 @@ def ci_high(self):
 
     def ci_valid(self) -> bool:
         """Return whether or not the result has valid confidence invervals"""
-        return self.ci_low() and self.ci_high()
+        return self.ci_low() and (not pd.isnull(self.ci_low())) and self.ci_high() and (not pd.isnull(self.ci_high()))
 
     def summary(self):
         """Summarise the causal test result as an intuitive sentence."""

tests/json_front_tests/test_json_class.py

@@ -123,7 +123,7 @@ def test_generate_coefficient_tests_from_json(self):
         # Test that the final log message prints that failed tests are printed, which is expected behaviour for this scenario
         with open("temp_out.txt", "r") as reader:
             temp_out = reader.readlines()
-        self.assertIn("failed", temp_out[-1])
+        self.assertIn("FAILED", temp_out[-1])
 
     def test_run_json_tests_from_json(self):
         example_test = {
@@ -234,7 +234,7 @@ def test_run_concrete_json_testcase(self):
         self.json_class.run_json_tests(effects=effects, estimators=estimators, f_flag=False)
         with open("temp_out.txt", "r") as reader:
             temp_out = reader.readlines()
-        self.assertIn("failed", temp_out[-1])
+        self.assertIn("FAILED", temp_out[-1])
 
     def tearDown(self) -> None:
         remove_temp_dir_if_existent()