Add decision rule & node id in scored dataset

Author: Agathe Guillemot

python-lib/dku_idtb_decision_tree/node.py

@@ -45,6 +45,9 @@ def set_node_info(self, samples, total_samples, probabilities, prediction):
     def get_type(self):
         raise NotImplementedError
 
+    def get_decision_rule(self):
+        raise NotImplementedError
+
     def rebuild(self, prediction, samples, probabilities):
         self.prediction = prediction
         self.samples = samples
@@ -67,6 +70,11 @@ def __init__(self, node_id, parent_id, treated_as_numerical, feature, values, ot
     def get_type(self):
         return Node.TYPES.CAT
 
+    def get_decision_rule(self):
+        return "{feature} {negation}in {values}".format(
+            feature=self.feature, negation="not " if self.others else "", values=self.values
+        )
+
     def apply_filter(self, df):
         if self.others:
             return df[~df[self.feature].isin(self.values)]
@@ -91,6 +99,15 @@ def __init__(self, node_id, parent_id, treated_as_numerical, feature, beginning=
     def get_type(self):
         return Node.TYPES.NUM
 
+    def get_decision_rule(self):
+        rule = ""
+        if self.beginning:
+            rule += "{} ≤ ".format(self.beginning)
+        rule += self.feature
+        if self.end:
+            rule += "< {}".format(self.end)
+        return rule
+
     def apply_filter(self, df, mean):
         if self.beginning is not None:
             df = df[df[self.feature].ge(self.beginning, fill_value=mean)]

python-lib/dku_idtb_decision_tree/tree.py

@@ -83,6 +83,14 @@ def add_node(self, node):
             parent_node.children_ids.append(node.id)
         super(ScoringTree, self).add_node(node)
 
+    def get_decision_rule(self, node_id):
+        rule = deque()
+        while node_id > 0:
+            node = self.get_node(node_id)
+            rule.appendleft(node.get_decision_rule())
+            node_id = node.parent_id
+        return list(rule)
+
 #Used by the webapp
 class InteractiveTree(Tree):
     """

python-lib/dku_idtb_scoring/score.py

@@ -18,25 +18,24 @@ def update_input_schema(input_schema, columns):
             new_input_schema.append(column)
     return new_input_schema
 
+def _add_column(name, type, schema, columns=None):
+    schema.append({'type': type, 'name': name})
+    if columns is not None:
+        columns.append(name)
+
 def get_scored_df_schema(tree, schema, columns, output_probabilities, is_evaluation=False, check_prediction=False):
     check_input_schema(tree, set(column["name"] for column in schema), is_evaluation)
     if columns is not None:
         schema = update_input_schema(schema, columns)
     if output_probabilities:
         for value in tree.target_values:
-            schema.append({'type': 'double', 'name': "proba_" + safe_str(value)})
-            if columns is not None:
-                columns.append("proba_"+safe_str(value))
-    schema.append({'type': 'string', 'name': 'prediction'})
-    if columns is not None:
-        columns.append("prediction")
+            _add_column('proba_' + safe_str(value), 'double', schema, columns)
+    _add_column('prediction', 'string', schema, columns)
     if check_prediction:
-        schema.append({'type': 'boolean', 'name': 'prediction_correct'})
-        if columns is not None:
-            columns.append("prediction_correct")
-    schema.append({'type': 'string', 'name': 'label'})
-    if columns is not None:
-        columns.append("label")
+        _add_column('prediction_correct', 'boolean', schema, columns)
+    _add_column('decision_rule', 'array', schema, columns)
+    _add_column('node_id', 'int', schema, columns)
+    _add_column('label', 'string', schema, columns)
     return schema
 
 def get_metric_df_schema(metrics_dict, metrics, recipe_config):
@@ -78,6 +77,7 @@ def add_scoring_columns(tree, df, output_probabilities, is_evaluation=False, che
                 df.loc[filtered_df_indices, "prediction_correct"] = filtered_df[tree.target] == leaf.prediction
             df.loc[label_indices, "label"] = leaf.label
 
-        elif leaf.label is not None:
-            filtered_df = tree.get_filtered_df(leaf, df)
-            df.loc[filtered_df.index, "label"] = leaf.label
+        filtered_df = tree.get_filtered_df(leaf, df)
+        df.loc[filtered_df.index, "decision_rule"] = safe_str(tree.get_decision_rule(leaf_id))
+        df.loc[filtered_df.index, "node_id"] = leaf_id
+        df.loc[filtered_df.index, "label"] = leaf.label