separating all features

Author: tawabshakeel

__init__.py

@@ -1 +1,3 @@
 from explainx.explain import *
+
+from explainx.main import *

demo-explainx-with-sound.gif

@@ -1,8 +1,6 @@
 import os
 import sys
-
 import re
-
 from pathlib import Path
 from sys import platform
 import subprocess
@@ -19,33 +17,25 @@
 from calculate_shap import *
 from analytics import Analytics
 
-"""
-This class calculates feature importance
-
-Input: 
-
-
-"""
-
-
 class explain():
     def __init__(self):
         super(explain, self).__init__()
         self.param = {}
 
     # is classification function?
 
-    def is_classification_given_y_array(self, y_test):
-        is_classification = False
-        total = len(y_test)
-        total_unique = len(set(y_test))
-        if total < 30:
-            if total_unique < 10:
-                is_classification = True
-        else:
-            if total_unique < 20:
-                is_classification = True
-        return is_classification
+    # def is_classification_given_y_array(self, y_test):
+    #     is_classification = False
+    #     total = len(y_test)
+    #     total_unique = len(set(y_test))
+    #     if total < 30:
+    #         if total_unique < 10:
+    #             is_classification = True
+    #     else:
+    #         if total_unique < 20:
+    #             is_classification = True
+    #     return is_classification
+    
 
     def random_string_generator(self):
         random_str = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
@@ -54,6 +44,8 @@ def random_string_generator(self):
     def ai(self, df, y, model, model_name="xgboost", mode=None):
         y_variable = "y_actual"
         y_variable_predict = "y_prediction"
+        
+        #Code for Analytics
         instance_id = self.random_string_generator()
         analytics = Analytics()
         analytics['ip'] = analytics.finding_ip()
@@ -69,11 +61,6 @@ def ai(self, df, y, model, model_name="xgboost", mode=None):
         analytics['finish_time'] = ''
         analytics.insert_data()
 
-        # If yes, then different shap functuions are required.
-        # get the shap value based on predcton and make a new dataframe.
-
-        # find predictions first as shap values need that.
-
         prediction_col = []
 
         if model_name == "xgboost":
@@ -88,40 +75,48 @@ def ai(self, df, y, model, model_name="xgboost", mode=None):
             prediction_col = model.predict(df.to_numpy())
 
         else:
-            prediction_col = model.predict(df.to_numpy())
+            prediction_col = model.predict(df)
 
         # is classification?
-        is_classification = self.is_classification_given_y_array(prediction_col)
+        #is_classification = self.is_classification_given_y_array(prediction_col)
+        ModelType = lambda model: True if is_classifier(model) else False
+        is_classification = ModelType(model)
 
         # shap
         c = calculate_shap()
         self.df_final, self.explainer = c.find(model, df, prediction_col, is_classification, model_name=model_name)
 
-        # prediction col
+        #Append Model Decision & True Labels Columns into the dataset.
         self.df_final[y_variable_predict] = prediction_col
-
         self.df_final[y_variable] = y
 
         # additional inputs.
         if is_classification == True:
             # find and add probabilities in the dataset.
-            prediction_col_prob = model.predict_proba(df.to_numpy())
-            pd_prediction_col_prob = pd.DataFrame(prediction_col_prob)
+            #prediction_col_prob = model.predict_proba(df)
+            #pd_prediction_col_prob = pd.DataFrame(prediction_col_prob)
 
-            for c in pd_prediction_col_prob.columns:
-                self.df_final["probability_of_predicting_class_" + str(c)] = list(pd_prediction_col_prob[c])
+            probabilities = model.predict_proba(df)
 
-            classes = []
-            for c in pd_prediction_col_prob.columns:
-                classes.append(str(c))
-            self.param["classes"] = classes
+            for i in range(len(np.unique(prediction_col))):
+                self.df_final['Probability: {}'.format(np.unique(prediction_col)[i])] = probabilities[:,i]
+            
+            self.param['classes'] = np.unique(prediction_col)
+
+            #for c in pd_prediction_col_prob.columns:
+             #   self.df_final["probability_of_predicting_class_" + str(c)] = list(pd_prediction_col_prob[c])
+
+            #classes = []
+            #for c in pd_prediction_col_prob.columns:
+             #   classes.append(str(c))
+            #self.param["classes"] = classes
 
             try:
                 expected_values_by_class = self.explainer.expected_value
             except:
                 expected_values_by_class = []
-                for c in range(len(classes)):
-                    expected_values_by_class.append(1 / len(classes))
+                for c in range(len(np.unique(prediction_col))):
+                    expected_values_by_class.append(1 / len(np.unique(prediction_col)))
 
             self.param["expected_values"] = expected_values_by_class
         else:

explain.py

@@ -9,27 +9,18 @@ def __init__(self):
 
     @staticmethod
     def finding_address():
-        try:
-            val = get_mac()
-            return val
-
-        except Exception as e :
-            return None
+        val = get_mac()
+        return val
 
     @staticmethod
     def finding_ip():
-        try:
-            val = socket.gethostbyname(socket.gethostname())
-            return val
-        except Exception as e:
-            return None
+        val = socket.gethostbyname(socket.gethostname())
+        return val
 
     @staticmethod
     def finding_system():
-        try:
-            return platform.system()
-        except Exception as e:
-            return None
+        return platform.system()
+
     def __setitem__(self, key, val):
         self.dict[key] = val
 

lib/analytics.py

@@ -9,7 +9,6 @@
 
 """
 
-
 class calculate_shap():
     def __init__(self):
         super(calculate_shap, self).__init__()

lib/calculate_shap.py

@@ -464,7 +464,6 @@ def toggle_collapse(n, is_open):
             return is_open
 
         #Cohort Analysis - Callbacks
-
         @app.callback(
         Output("modal", "is_open"),
         [Input("open", "n_clicks"), Input("close", "n_clicks")],
@@ -675,6 +674,7 @@ def update_graph(xaxis_column_name, third_axis_name, sql_query):
             g = plotly_graphs()
             graph_type = 'pdp'
             df3 = self.caching_data_manager(df, sql_query, graph_type, g.partial_dependence_plot)
+            print(df3)
             fig = g.pdp_plot(df3, df3[xaxis_column_name], df3[xaxis_column_name+"_impact"], df3[third_axis_name])
             return fig
 

lib/dashboard.py

@@ -1,13 +1,7 @@
 from imports import *
 from sklearn.preprocessing import OneHotEncoder
 import numpy as np
-"""
-This class calculates feature importance
 
-Input: 
-
-
-"""
 
 
 class encode_decode_cat_col():

lib/encode_decode_cat_col.py

@@ -1,22 +1,14 @@
 from imports import *
 
-"""
-This class calculates feature impact
-
-Input: 
-
-
-"""
-
-
 class feature_impact():
     def __init__(self):
         super(feature_impact, self).__init__()
         self.param= None
 
 
     def find(self,  df):
-
+        df  = pd.DataFrame(df)
+        print(df)
         variables = [col for col in df.columns if '_impact' in col]
         y = []
         for i in range(len(variables)):

lib/feature_impact.py

@@ -1,14 +1,5 @@
 from imports import *
 
-"""
-This class calculates feature impact
-
-Input: 
-
-
-"""
-
-
 class feature_impact_classification():
     def __init__(self):
         super(feature_impact_classification, self).__init__()

lib/feature_impact_classification.py

@@ -1,14 +1,5 @@
 from imports import *
 
-"""
-This class calculates feature importance
-
-Input: 
-
-
-"""
-
-
 class feature_importance():
     def __init__(self):
         super(feature_importance, self).__init__()