Implemented and tested MR generation from dag method

Author: AndrewC19

causal_testing/specification/metamorphic_relation.py

@@ -4,10 +4,12 @@
 from itertools import combinations
 import numpy as np
 import pandas as pd
+import networkx as nx
 
 from causal_testing.specification.causal_specification import CausalDAG, Node
 from causal_testing.data_collection.data_collector import ExperimentalDataCollector
 
+
 @dataclass(order=True)
 class MetamorphicRelation:
     """Class representing a metamorphic relation."""
@@ -89,12 +91,12 @@ def execute_tests(self, data_collector: ExperimentalDataCollector):
             data_collector.control_input_configuration = control_input_config
             data_collector.treatment_input_configuration = treatment_input_config
             metamorphic_test_results_df = data_collector.collect_data()
+
             # Apply assertion to control and treatment outputs
             control_output = metamorphic_test_results_df.loc["control_0"][metamorphic_test.output]
             treatment_output = metamorphic_test_results_df.loc["treatment_0"][metamorphic_test.output]
+
             if not self.assertion(control_output, treatment_output):
-                print(metamorphic_test.output)
-                print(control_output, treatment_output)
                 test_results["fail"].append(metamorphic_test)
             else:
                 test_results["pass"].append(metamorphic_test)
@@ -166,3 +168,43 @@ def __str__(self):
                f"Other inputs: {self.other_inputs}\n" \
                f"Output: {self.output}" \
                f"Metamorphic Relation: {self.relation}"
+
+
+def generate_metamorphic_relations(dag: CausalDAG) -> list[MetamorphicRelation]:
+    """Construct a list of metamorphic relations implied by the Causal DAG.
+
+    This list of metamorphic relations contains a ShouldCause relation for every edge, and a ShouldNotCause
+    relation for every conditional independence relation.
+    """
+    metamorphic_relations = []
+    for node_pair in combinations(dag.graph.nodes, 2):
+        (u, v) = node_pair
+
+        # Create a ShouldNotCause relation for each pair of nodes that are not directly connected
+        if ((u, v) not in dag.graph.edges) and ((v, u) not in dag.graph.edges):
+
+            # Case 1: U --> ... --> V
+            if u in nx.ancestors(dag.graph, v):
+                adj_set = list(dag.direct_effect_adjustment_sets([u], [v])[0])
+                metamorphic_relations.append(ShouldNotCause(u, v, adj_set, dag))
+
+            # Case 2: V --> ... --> U
+            elif v in nx.ancestors(dag.graph, u):
+                adj_set = list(dag.direct_effect_adjustment_sets([v], [u])[0])
+                metamorphic_relations.append(ShouldNotCause(v, u, adj_set, dag))
+
+            # Case 3: V _||_ U (neither is a predecessor)
+            # Only make one MR since V _||_ U == U _||_ V
+            else:
+                adj_set = list(dag.direct_effect_adjustment_sets([u], [v])[0])
+                metamorphic_relations.append(ShouldNotCause(u, v, adj_set, dag))
+
+        # Create a ShouldCause relation for each edge (u, v) or (v, u)
+        elif (u, v) in dag.graph.edges:
+            adj_set = list(dag.direct_effect_adjustment_sets([u], [v])[0])
+            metamorphic_relations.append(ShouldCause(u, v, adj_set, dag))
+        else:
+            adj_set = list(dag.direct_effect_adjustment_sets([v], [u])[0])
+            metamorphic_relations.append(ShouldCause(v, u, adj_set, dag))
+
+    return metamorphic_relations

tests/specification_tests/test_metamorphic_relations.py

@@ -1,12 +1,14 @@
 import unittest
 import os
+
 import pandas as pd
 from itertools import combinations
 
-from tests.test_helpers import create_temp_dir_if_non_existent, remove_temp_dir_if_existent
+from tests.test_helpers import create_temp_dir_if_non_existent
 from causal_testing.specification.causal_dag import CausalDAG
 from causal_testing.specification.causal_specification import Scenario
-from causal_testing.specification.metamorphic_relation import ShouldCause, ShouldNotCause
+from causal_testing.specification.metamorphic_relation import (ShouldCause, ShouldNotCause,
+                                                               generate_metamorphic_relations)
 from causal_testing.data_collection.data_collector import ExperimentalDataCollector
 from causal_testing.specification.variable import Input, Output
 
@@ -74,7 +76,8 @@ def test_should_cause_metamorphic_relations_correct_spec(self):
         causal_dag = CausalDAG(self.dag_dot_path)
         for edge in causal_dag.graph.edges:
             (u, v) = edge
-            should_cause_MR = ShouldCause(u, v, None, causal_dag)
+            adj_set = list(causal_dag.direct_effect_adjustment_sets([u], [v])[0])
+            should_cause_MR = ShouldCause(u, v, adj_set, causal_dag)
             should_cause_MR.generate_follow_up(10, -10.0, 10.0, 1)
             test_results = should_cause_MR.execute_tests(self.data_collector)
             should_cause_MR.test_oracle(test_results)
@@ -88,9 +91,10 @@ def test_should_not_cause_metamorphic_relations_correct_spec(self):
             if ((u, v) not in causal_dag.graph.edges) and ((v, u) not in causal_dag.graph.edges):
                 # Check both directions if there is no causality
                 # This can be done more efficiently by ignoring impossible directions (output --> input)
-                adj_set = list(causal_dag.direct_effect_adjustment_sets([u], [v])[0])
-                u_should_not_cause_v_MR = ShouldNotCause(u, v, adj_set, causal_dag)
-                v_should_not_cause_u_MR = ShouldNotCause(v, u, adj_set, causal_dag)
+                adj_set_u_to_v = list(causal_dag.direct_effect_adjustment_sets([u], [v])[0])
+                u_should_not_cause_v_MR = ShouldNotCause(u, v, adj_set_u_to_v, causal_dag)
+                adj_set_v_to_u = list(causal_dag.direct_effect_adjustment_sets([v], [u])[0])
+                v_should_not_cause_u_MR = ShouldNotCause(v, u, adj_set_v_to_u, causal_dag)
                 u_should_not_cause_v_MR.generate_follow_up(10, -100, 100)
                 v_should_not_cause_u_MR.generate_follow_up(10, -100, 100)
                 u_should_not_cause_v_test_results = u_should_not_cause_v_MR.execute_tests(self.data_collector)
@@ -115,4 +119,41 @@ def test_should_cause_metamorphic_relation_missing_relationship(self):
         self.assertRaises(AssertionError, X1_should_cause_Z_MR.test_oracle, X1_should_cause_Z_test_results)
         self.assertRaises(AssertionError, X2_should_cause_Z_MR.test_oracle, X2_should_cause_Z_test_results)
 
-
+    def test_all_metamorphic_relations_implied_by_dag(self):
+        dag = CausalDAG(self.dag_dot_path)
+        metamorphic_relations = generate_metamorphic_relations(dag)
+        should_cause_relations = [mr for mr in metamorphic_relations if isinstance(mr, ShouldCause)]
+        should_not_cause_relations = [mr for mr in metamorphic_relations if isinstance(mr, ShouldNotCause)]
+
+        # Check all ShouldCause relations are present and no extra
+        expected_should_cause_relations = [ShouldCause('X1', 'Z', [], dag),
+                                           ShouldCause('Z', 'M', [], dag),
+                                           ShouldCause('M', 'Y', [], dag),
+                                           ShouldCause('X2', 'Z', [], dag),
+                                           ShouldCause('X3', 'M', [], dag)]
+
+        extra_sc_relations = [scr for scr in should_cause_relations if scr not in expected_should_cause_relations]
+        missing_sc_relations = [escr for escr in expected_should_cause_relations if escr not in should_cause_relations]
+
+        self.assertEqual(extra_sc_relations, [])
+        self.assertEqual(missing_sc_relations, [])
+
+        # Check all ShouldNotCause relations are present and no extra
+        expected_should_not_cause_relations = [ShouldNotCause('X1', 'X2', [], dag),
+                                               ShouldNotCause('X1', 'X3', [], dag),
+                                               ShouldNotCause('X1', 'M', ['Z'], dag),
+                                               ShouldNotCause('X1', 'Y', ['M'], dag),
+                                               ShouldNotCause('X2', 'X3', [], dag),
+                                               ShouldNotCause('X2', 'M', ['Z'], dag),
+                                               ShouldNotCause('X2', 'Y', ['M'], dag),
+                                               ShouldNotCause('X3', 'Y', ['M'], dag),
+                                               ShouldNotCause('Z', 'Y', ['M'], dag),
+                                               ShouldNotCause('Z', 'X3', [], dag)]
+
+        extra_snc_relations = [sncr for sncr in should_not_cause_relations if sncr
+                               not in expected_should_not_cause_relations]
+        missing_snc_relations = [esncr for esncr in expected_should_not_cause_relations if esncr
+                                 not in should_not_cause_relations]
+
+        self.assertEqual(extra_snc_relations, [])
+        self.assertEqual(missing_snc_relations, [])