Merge pull request #431 from vprashar2929/add-hatch-fmt

feat(ci): Integrate hatch fmt for code formatting

Author: Sunil Thaha

.github/pull_request_template.md

@@ -7,7 +7,7 @@ In addition to approval, the author must confirm the following check list:
 - [ ] Run the following command to format your code:
 
   ```bash
-  hatch fmt
+  make fmt
   ```
 
 - [ ] Create issues for unresolved comments and link them to this PR. Use one of the following labels:

.github/workflows/lint.yml

@@ -28,3 +28,10 @@ jobs:
         shell: bash
         run: |
           make lint
+
+      # run hatch fmt
+      - name: Run formatter using hatch
+        shell: bash
+        run: |
+          make fmt
+          git diff --exit-code

Makefile

@@ -15,6 +15,10 @@ PYTHON  := python3.10
 DOCKERFILES_PATH := ./dockerfiles
 MODEL_PATH := ${PWD}/tests/models
 
+.PHONY: fmt
+fmt:
+	@hatch fmt -f
+
 .PHONY: lint
 lint:
 	@hatch run pymarkdownlnt scan -r .

model_training/s3/src/s3/loader.py

@@ -58,7 +58,9 @@ def aws_download(client, bucket_name, machine_id, mnt_path, pipeline_name):
 
 def ibm_download(client, bucket_name, machine_id, mnt_path, pipeline_name):
     print("IBM Download")
-    bucket_file_map = get_bucket_file_map(client, bucket_name, machine_id=machine_id, mnt_path=mnt_path, pipeline_name=pipeline_name, list_func=ibmcloud_list_keys)
+    bucket_file_map = get_bucket_file_map(
+        client, bucket_name, machine_id=machine_id, mnt_path=mnt_path, pipeline_name=pipeline_name, list_func=ibmcloud_list_keys
+    )
     for key, filepath in bucket_file_map.items():
         print(key, filepath)
         dir = os.path.dirname(filepath)

model_training/s3/src/s3/util.py

@@ -7,7 +7,13 @@ def new_ibm_client(args):
     import ibm_boto3
     from ibm_botocore.client import Config
 
-    cos = ibm_boto3.resource("s3", ibm_api_key_id=args.api_key, ibm_service_instance_id=args.service_instance_id, config=Config(signature_version="oauth"), endpoint_url=args.service_endpoint)
+    cos = ibm_boto3.resource(
+        "s3",
+        ibm_api_key_id=args.api_key,
+        ibm_service_instance_id=args.service_instance_id,
+        config=Config(signature_version="oauth"),
+        endpoint_url=args.service_endpoint,
+    )
     return cos
 
 

pyproject.toml

@@ -111,7 +111,7 @@ exclude_lines = [
 ]
 
 [tool.ruff]
-line-length = 320
+line-length = 160
 
 [tool.pytest.ini_options]
 markers = [

src/kepler_model/cmd/cmd_plot.py

@@ -140,7 +140,9 @@ def metadata_plot(args, energy_source, metadata_df, output_folder, name):
             ax = axes
         else:
             ax = axes[i]
-        sns.barplot(data=metadata_df, x="feature_group", y="mae", hue="trainer", ax=ax, hue_order=sorted(metadata_df["trainer"].unique()), errorbar=None, palette="Set3")
+        sns.barplot(
+            data=metadata_df, x="feature_group", y="mae", hue="trainer", ax=ax, hue_order=sorted(metadata_df["trainer"].unique()), errorbar=None, palette="Set3"
+        )
         ax.set_title(component)
         ax.set_ylabel("MAE (Watt)")
         ax.set_xlabel("Feature Group")
@@ -166,7 +168,9 @@ def power_curve_plot(args, data_path, energy_source, output_folder, name):
     node_collection = NodeTypeIndexCollection(pipeline_path)
     all_node_types = sorted(list(node_collection.node_type_index.keys()))
     output_type = ModelOutputType[args.output_type]
-    models, _, cpu_ms_max = _load_all_models(model_toppath=model_toppath, output_type=output_type, name=pipeline_name, node_types=all_node_types, energy_source=energy_source)
+    models, _, cpu_ms_max = _load_all_models(
+        model_toppath=model_toppath, output_type=output_type, name=pipeline_name, node_types=all_node_types, energy_source=energy_source
+    )
     if len(models) > 0:
         _plot_models(models, cpu_ms_max, energy_source, output_folder, name)
 

src/kepler_model/cmd/cmd_util.py

@@ -316,7 +316,9 @@ def get_isolator(data_path, isolator, profile, pipeline_name, target_hints, bg_h
         profile_isolator = ProfileBackgroundIsolator(profiles, idle_data)
         supported_isolator[profile_isolator.get_name()] = profile_isolator
         if abs_pipeline_name != "":
-            trainer_isolator = TrainIsolator(idle_data=idle_data, profiler=DefaultProfiler, target_hints=target_hints, bg_hints=bg_hints, abs_pipeline_name=abs_pipeline_name)
+            trainer_isolator = TrainIsolator(
+                idle_data=idle_data, profiler=DefaultProfiler, target_hints=target_hints, bg_hints=bg_hints, abs_pipeline_name=abs_pipeline_name
+            )
             supported_isolator[trainer_isolator.get_name()] = trainer_isolator
     elif abs_pipeline_name != "":
         trainer_isolator = TrainIsolator(target_hints=target_hints, bg_hints=bg_hints, abs_pipeline_name=abs_pipeline_name)
@@ -338,10 +340,32 @@ def get_extractor(extractor):
     return supported_extractor[extractor]
 
 
-def get_pipeline(data_path, pipeline_name, extractor, profile, target_hints, bg_hints, abs_pipeline_name, isolator, abs_trainer_names, dyn_trainer_names, energy_sources, valid_feature_groups, replace_node_type=default_node_type):
+def get_pipeline(
+    data_path,
+    pipeline_name,
+    extractor,
+    profile,
+    target_hints,
+    bg_hints,
+    abs_pipeline_name,
+    isolator,
+    abs_trainer_names,
+    dyn_trainer_names,
+    energy_sources,
+    valid_feature_groups,
+    replace_node_type=default_node_type,
+):
     from kepler_model.train import NewPipeline
 
     isolator = get_isolator(data_path, isolator, profile, pipeline_name, target_hints, bg_hints, abs_pipeline_name, replace_node_type=replace_node_type)
     extractor = get_extractor(extractor)
-    pipeline = NewPipeline(pipeline_name, abs_trainer_names, dyn_trainer_names, extractor=extractor, isolator=isolator, target_energy_sources=energy_sources, valid_feature_groups=valid_feature_groups)
+    pipeline = NewPipeline(
+        pipeline_name,
+        abs_trainer_names,
+        dyn_trainer_names,
+        extractor=extractor,
+        isolator=isolator,
+        target_energy_sources=energy_sources,
+        valid_feature_groups=valid_feature_groups,
+    )
     return pipeline

src/kepler_model/cmd/main.py

@@ -458,7 +458,20 @@ def train(args):
     pipeline = None
     if args.id:
         machine_id = args.id
-        pipeline = get_pipeline(data_path, pipeline_name, args.extractor, args.profile, args.target_hints, args.bg_hints, args.abs_pipeline_name, args.isolator, abs_trainer_names, dyn_trainer_names, energy_sources, valid_feature_groups)
+        pipeline = get_pipeline(
+            data_path,
+            pipeline_name,
+            args.extractor,
+            args.profile,
+            args.target_hints,
+            args.bg_hints,
+            args.abs_pipeline_name,
+            args.isolator,
+            abs_trainer_names,
+            dyn_trainer_names,
+            energy_sources,
+            valid_feature_groups,
+        )
         machine_spec_json = load_machine_spec(data_path, machine_id)
         if machine_spec_json is not None:
             new_spec = NodeTypeSpec()
@@ -475,7 +488,9 @@ def train(args):
     for energy_source in energy_sources:
         energy_components = PowerSourceMap[energy_source]
         for feature_group in valid_feature_groups:
-            success, abs_data, dyn_data = pipeline.process_multiple_query(input_query_results_list, energy_components, energy_source, feature_group=feature_group.name, replace_node_type=node_type)
+            success, abs_data, dyn_data = pipeline.process_multiple_query(
+                input_query_results_list, energy_components, energy_source, feature_group=feature_group.name, replace_node_type=node_type
+            )
             assert success, f"failed to process pipeline {pipeline.name}"
             for trainer in pipeline.trainers:
                 if trainer.feature_group == feature_group and trainer.energy_source == energy_source:
@@ -488,7 +503,11 @@ def train(args):
             if abs_data is not None:
                 save_csv(data_saved_path, get_general_filename("preprocess", energy_source, feature_group, ModelOutputType.AbsPower, args.extractor), abs_data)
             if dyn_data is not None:
-                save_csv(data_saved_path, get_general_filename("preprocess", energy_source, feature_group, ModelOutputType.DynPower, args.extractor, args.isolator), dyn_data)
+                save_csv(
+                    data_saved_path,
+                    get_general_filename("preprocess", energy_source, feature_group, ModelOutputType.DynPower, args.extractor, args.isolator),
+                    dyn_data,
+                )
 
         print(f"=========== Train {energy_source} Summary ============")
         # save args
@@ -580,7 +599,20 @@ def estimate(args):
             if pipeline_metadata is None:
                 print(f"no metadata for pipeline {pipeline_name}.")
                 continue
-            pipeline = get_pipeline(data_path, pipeline_name, pipeline_metadata["extractor"], args.profile, args.target_hints, args.bg_hints, args.abs_pipeline_name, pipeline_metadata["isolator"], pipeline_metadata["abs_trainers"], pipeline_metadata["dyn_trainers"], energy_sources, valid_fg)
+            pipeline = get_pipeline(
+                data_path,
+                pipeline_name,
+                pipeline_metadata["extractor"],
+                args.profile,
+                args.target_hints,
+                args.bg_hints,
+                args.abs_pipeline_name,
+                pipeline_metadata["isolator"],
+                pipeline_metadata["abs_trainers"],
+                pipeline_metadata["dyn_trainers"],
+                energy_sources,
+                valid_fg,
+            )
             if pipeline is None:
                 print(f"cannot get pipeline {pipeline_name}.")
                 continue
@@ -742,7 +774,16 @@ def plot(args):
                 predicted_power_cols += [predicted_power_colname]
             data_filename = get_general_filename(args.target_data, energy_source, fg, ot, args.extractor, args.isolator)
             # plot prediction
-            ts_plot(data, cols, f"{energy_source} {ot.name} Prediction Result \n by {model_id}", output_folder, f"{data_filename}_{model_id}", subtitles=subtitles, labels=plot_labels, ylabel="Power (W)")
+            ts_plot(
+                data,
+                cols,
+                f"{energy_source} {ot.name} Prediction Result \n by {model_id}",
+                output_folder,
+                f"{data_filename}_{model_id}",
+                subtitles=subtitles,
+                labels=plot_labels,
+                ylabel="Power (W)",
+            )
             # plot correlation to utilization if feature group is set
             if fg is not None:
                 feature_cols = FeatureGroups[fg]
@@ -751,7 +792,17 @@ def plot(args):
                 # plot raw feature data to confirm min-max value
                 ts_plot(data, feature_cols, f"Features {fg}", output_folder, f"{data_filename}_{fg}", labels=None, subtitles=None, ylabel=None)
                 data[feature_cols] = scaler.fit_transform(data[feature_cols])
-                feature_power_plot(data, model_id, ot.name, energy_source, feature_cols, actual_power_cols, predicted_power_cols, output_folder, f"{data_filename}_{model_id}_corr")
+                feature_power_plot(
+                    data,
+                    model_id,
+                    ot.name,
+                    energy_source,
+                    feature_cols,
+                    actual_power_cols,
+                    predicted_power_cols,
+                    output_folder,
+                    f"{data_filename}_{model_id}_corr",
+                )
 
     elif args.target_data == "error":
         from sklearn.preprocessing import MaxAbsScaler
@@ -964,7 +1015,9 @@ def run():
     parser.add_argument("-e", "--energy-source", type=str, help="Specify energy source.", default="rapl-sysfs")
     parser.add_argument("--abs-trainers", type=str, help="Specify trainer names for train command (use comma(,) as delimiter).", default="default")
     parser.add_argument("--dyn-trainers", type=str, help="Specify trainer names for train command (use comma(,) as delimiter).", default="default")
-    parser.add_argument("--trainers", type=str, help="Specify trainer names for train_from_data command (use comma(,) as delimiter).", default="XgboostFitTrainer")
+    parser.add_argument(
+        "--trainers", type=str, help="Specify trainer names for train_from_data command (use comma(,) as delimiter).", default="XgboostFitTrainer"
+    )
 
     # Validate arguments
     parser.add_argument("--benchmark", type=str, help="Specify benchmark file name.")

src/kepler_model/estimate/archived_model.py

@@ -87,4 +87,3 @@ def get_achived_model(power_request):
             print("cannot validate the archived model")
             return None
     return output_path
-