Merge pull request #146 from marceloamaral/abs-scaler

Update the feature normalization to use maxAbsScaler

Author: sunya-ch

cmd/main.py

@@ -652,7 +652,7 @@ def plot(args):
                     _ts_plot(power_data, power_cols, "Power source: {}".format(energy_source), output_folder, data_filename, ylabel="Power (W)")
     elif args.target_data == "estimate":
         from estimate import default_predicted_col_func
-        from sklearn.preprocessing import MinMaxScaler
+        from sklearn.preprocessing import MaxAbsScaler
 
         best_result_map, power_labels_map, best_model_id_map, _ = estimate(args)
         for energy_source, best_restult in best_result_map.items():
@@ -680,13 +680,13 @@ def plot(args):
             # plot correlation to utilization if feature group is set
             if fg is not None:
                 feature_cols = FeatureGroups[fg]
-                scaler = MinMaxScaler()
+                scaler = MaxAbsScaler()
                 data[feature_cols] = best_restult[[TIMESTAMP_COL] + feature_cols].groupby([TIMESTAMP_COL]).sum().sort_index()
                 data[feature_cols] = scaler.fit_transform(data[feature_cols])
                 _feature_power_plot(data, model_id, ot.name, energy_source, feature_cols, actual_power_cols, predicted_power_cols, output_folder, "{}_{}_corr".format(data_filename, model_id))
     elif args.target_data == "error":
         from estimate import default_predicted_col_func
-        from sklearn.preprocessing import MinMaxScaler
+        from sklearn.preprocessing import MaxAbsScaler
         _, _, _, summary_df = estimate(args)
         for energy_source in energy_sources:
             data_filename = get_general_filename(args.target_data, energy_source, fg, ot, args.extractor, args.isolator)

src/train/profiler/generate_scaler.py

@@ -25,7 +25,7 @@
 sys.path.append(util_path)
 sys.path.append(tool_path)
 
-from sklearn.preprocessing import MinMaxScaler, StandardScaler
+from sklearn.preprocessing import MaxAbsScaler
 
 from train import DefaultExtractor, node_info_column, FeatureGroups, FeatureGroup, TIMESTAMP_COL
 from util.train_types import SYSTEM_FEATURES
@@ -36,13 +36,10 @@
 
 extractor = DefaultExtractor()
 
-minmax_scaler_top_path = os.path.join(profile_path, '..', 'minmax_scaler')
-standard_scaler_top_path = os.path.join(profile_path, '..', 'standard_scaler')
+max_scaler_top_path = os.path.join(profile_path, '..', 'max_scaler')
 
-if not os.path.exists(minmax_scaler_top_path):
-    os.mkdir(minmax_scaler_top_path)
-if not os.path.exists(standard_scaler_top_path):
-    os.mkdir(standard_scaler_top_path)
+if not os.path.exists(max_scaler_top_path):
+    os.mkdir(max_scaler_top_path)
 
 def read_query_results(query_path):
     results = dict()
@@ -88,9 +85,6 @@ def process(query_results):
             node_types = pd.unique(feature_data[node_info_column])
             # filter and extract features
             x_values = feature_data[feature_data[node_info_column]==node_type][features].values
-            standard_scaler = StandardScaler()
-            minmax_scaler = MinMaxScaler()
-            standard_scaler.fit(x_values)
-            minmax_scaler.fit(x_values)
-            save_scaler(standard_scaler, node_type, feature_group_name, standard_scaler_top_path)
-            save_scaler(minmax_scaler, node_type, feature_group_name, minmax_scaler_top_path)
+            max_scaler = MaxAbsScaler()
+            max_scaler.fit(x_values)
+            save_scaler(max_scaler, node_type, feature_group_name, max_scaler_top_path)

src/train/profiler/profiler.py

@@ -137,28 +137,23 @@ def __init__(self, node_type):
         self.profile = dict()
         for source in PowerSourceMap.keys():
             self.profile[source] = dict()
-        self.standard_scaler = dict()
-        self.minmax_scaler = dict()
+        self.max_scaler = dict()
         for feature_group in FeatureGroups.keys():
             #######################
             # uncomment to append from remote profile
             #
             # feature_key = feature_group.name
-            # standard_scaler = Profile.load_scaler(self.node_type, feature_key, scaler_type="standard")
-            # minmax_scaler = Profile.load_scaler(self.node_type, feature_key, scaler_type="minmax")
+            # max_scaler = Profile.load_scaler(self.node_type, feature_key, scaler_type="maxabs")
             #######################
-            standard_scaler = None
-            minmax_scaler = None
-            if standard_scaler is not None:
-                self.standard_scaler[feature_group.name] = standard_scaler
-            if minmax_scaler is not None:
-                self.minmax_scaler[feature_group.name] = minmax_scaler
+            max_scaler = None
+            if max_scaler is not None:
+                self.max_scaler[feature_group.name] = max_scaler
 
     def add_profile(self, source, component, profile_value):
         self.profile[source][component] = profile_value
 
     @staticmethod
-    def load_scaler(node_type, feature_key, scaler_type): # scaler_type = minmax or standard
+    def load_scaler(node_type, feature_key, scaler_type): # scaler_type = maxabs
         try:
             url_path = os.path.join(profiler_registry, scaler_type + "_scaler", str(node_type), feature_key + ".pkl")
             response = urlopen(url_path)
@@ -168,15 +163,10 @@ def load_scaler(node_type, feature_key, scaler_type): # scaler_type = minmax or
             print(url_path, e)
             return None
 
-    def get_minmax_scaler(self, feature_key):
-        if feature_key not in self.minmax_scaler:
+    def get_max_scaler(self, feature_key):
+        if feature_key not in self.max_scaler:
             return None
-        return self.minmax_scaler[feature_key]
-
-    def get_standard_scaler(self, feature_key):
-        if feature_key not in self.standard_scaler:
-            return None
-        return self.standard_scaler[feature_key]
+        return self.max_scaler[feature_key]
 
     def get_background_power(self, source, component):
         if source not in self.profile:
@@ -191,7 +181,7 @@ def get_min_power(self, source, component):
         return self.profile[source][component][min_watt_key]
 
     def print_profile(self):
-        print("Profile (node type={}): \n Available energy components: {}\n Available minmax scalers: {}\n Available standard scalers: {}".format(self.node_type, ["{}/{}".format(key, list(self.profile[key].keys())) for key in self.profile.keys()], self.minmax_scaler.keys(), self.standard_scaler.keys()))
+        print("Profile (node type={}): \n Available energy components: {}\n Available maxabs scalers: {}".format(self.node_type, ["{}/{}".format(key, list(self.profile[key].keys())) for key in self.profile.keys()], self.max_scaler.keys()))
 
 def generate_profiles(profile_map):
     profiles = dict()

src/train/trainer/SGDRegressorTrainer/main.py

@@ -1,5 +1,5 @@
 from sklearn.linear_model import SGDRegressor
-from sklearn.preprocessing import StandardScaler
+from sklearn.preprocessing import MaxAbsScaler
 import joblib
 from urllib.request import urlopen
 
@@ -12,7 +12,7 @@
 
 class SGDRegressorTrainer(ScikitTrainer):
     def __init__(self, energy_components, feature_group, energy_source, node_level, pipeline_name):
-        super(SGDRegressorTrainer, self).__init__(energy_components, feature_group, energy_source, node_level, scaler_type="standard", pipeline_name=pipeline_name)
+        super(SGDRegressorTrainer, self).__init__(energy_components, feature_group, energy_source, node_level, pipeline_name=pipeline_name)
         self.fe_files = []
 
     def init_model(self):

src/train/trainer/SVRRegressorTrainer/main.py

@@ -1,4 +1,4 @@
-from sklearn.preprocessing import StandardScaler
+from sklearn.preprocessing import MaxAbsScaler
 from sklearn.svm import SVR
 from sklearn.pipeline import make_pipeline
 
@@ -12,7 +12,7 @@
 
 class SVRRegressorTrainer(ScikitTrainer):
     def __init__(self, energy_components, feature_group, energy_source, node_level, pipeline_name):
-        super(SVRRegressorTrainer, self).__init__(energy_components, feature_group, energy_source, node_level, scaler_type="standard", pipeline_name=pipeline_name)
+        super(SVRRegressorTrainer, self).__init__(energy_components, feature_group, energy_source, node_level, pipeline_name=pipeline_name)
         self.fe_files = []
 
     def init_model(self):

src/train/trainer/__init__.py

@@ -23,7 +23,7 @@ def get_assured_checkpoint_path(group_path, assure=True):
 
 import pandas as pd
 from sklearn.model_selection import train_test_split
-from sklearn.preprocessing import MinMaxScaler, StandardScaler
+from sklearn.preprocessing import MaxAbsScaler
 
 def normalize_and_split(X_values, y_values, scaler, test_size=0.1):
     features = scaler.transform(X_values)
@@ -32,7 +32,7 @@ def normalize_and_split(X_values, y_values, scaler, test_size=0.1):
 
 
 class Trainer(metaclass=ABCMeta):
-    def __init__(self, model_class, energy_components, feature_group, energy_source, node_level, pipeline_name, scaler_type="minmax"):
+    def __init__(self, model_class, energy_components, feature_group, energy_source, node_level, pipeline_name, scaler_type="maxabs"):
         self.energy_components = energy_components
         self.feature_group_name = feature_group
         self.feature_group = FeatureGroup[feature_group]
@@ -133,10 +133,7 @@ def process(self, data, power_labels, pipeline_lock):
                 self.print_log("fit scaler to latest data".format(node_type, self.feature_group_name))
                 # no profiled scaler
                 x_values = node_type_filtered_data[self.features].values
-                if self.scaler_type == "standard":
-                    self.node_scalers[node_type] = StandardScaler()
-                else:
-                    self.node_scalers[node_type] = MinMaxScaler()
+                self.node_scalers[node_type] = MaxAbsScaler()
                 self.node_scalers[node_type].fit(x_values)
 
             X_test_map = dict()

src/train/trainer/scikit.py

@@ -16,8 +16,7 @@ def get_save_path(model_filepath):
     return "/".join(model_filepath.split("/")[0:-1])
 
 class ScikitTrainer(Trainer):
-    def __init__(self, energy_components, feature_group, energy_source, node_level, pipeline_name, scaler_type="minmax"):
-        self.is_standard_scaler = scaler_type == "standard"
+    def __init__(self, energy_components, feature_group, energy_source, node_level, pipeline_name, scaler_type="maxabs"):
         super(ScikitTrainer, self).__init__(model_class, energy_components, feature_group, energy_source, node_level, pipeline_name, scaler_type=scaler_type)
         self.fe_files = []
 
@@ -64,24 +63,23 @@ def save_model(self, component_save_path, node_type, component):
     def component_model_filename(self, component):
         return component + ".pkl"
 
-    def get_weight_dict(self, node_type):
-        if not self.is_standard_scaler:
-            # cannot get weight dict
-            return None
+    def get_weight_dict(self, node_type):    
         weight_dict = dict()
 
         for component, model in self.node_models[node_type].items():
             scaler = self.node_scalers[node_type]
             if not hasattr(model, "intercept_") or not hasattr(model, "coef_") or len(model.coef_) != len(self.features) or len(model.intercept_) != 1:
                 return None
             else:
+                # TODO: remove the mean and variance variables after updating the Kepler code
                 weight_dict[component] = {
                     "All_Weights": {
                         "Bias_Weight": model.intercept_[0],
                         "Categorical_Variables": dict(),
                         "Numerical_Variables": {self.features[i]: 
-                                                {"mean": scaler.mean_[i], 
-                                                "variance": scaler.var_[i], 
+                                                {"scale": scaler.scale_[i],
+                                                "mean": 0, 
+                                                "variance": 0, 
                                                 "weight": model.coef_[i], 
                                                 }
                                                 for i in range(len(self.features))},

tests/common_plot.py

@@ -19,7 +19,7 @@
 from util.prom_types import TIMESTAMP_COL
 from util.extract_types import col_to_component
 
-from sklearn.preprocessing import MinMaxScaler
+from sklearn.preprocessing import MaxAbsScaler
 
 from train.extractor.preprocess import get_extracted_power_labels
 from estimate import get_label_power_colname
@@ -31,7 +31,7 @@ def _fig_filename(figname, save_path=plot_output_path):
     return os.path.join(save_path, figname + ".png")
 
 def preprocess_data(df):
-    scaler = MinMaxScaler()
+    scaler = MaxAbsScaler()
     df = df.reset_index()
     normalized_data = scaler.fit_transform(df.values)
     normalized_df = pd.DataFrame(normalized_data, columns=df.columns, index=df.index)