Merge pull request #383 from sunya-ch/estimator

fix: mismatch model request check (unexpectedly-repeated model request)

Author: Sunil Thaha

manifests/base/patch/patch-estimator-sidecar.yaml

@@ -6,9 +6,9 @@ metadata:
 data:
   MODEL_CONFIG: |
     NODE_COMPONENTS_ESTIMATOR=true
-    NODE_COMPONENTS_INIT_URL=https://raw.githubusercontent.com/sustainable-computing-io/kepler-model-db/main/models/v0.7/ec2-0.7.11/rapl-sysfs/AbsPower/BPFOnly/GradientBoostingRegressorTrainer_0.zip
+    NODE_COMPONENTS_INIT_URL=https://raw.githubusercontent.com/sustainable-computing-io/kepler-model-db/main/models/v0.7/ec2-0.7.11/rapl-sysfs/AbsPower/BPFOnly/SGDRegressorTrainer_0.zip
     NODE_TOTAL_ESTIMATOR=true
-    NODE_TOTAL_INIT_URL=https://raw.githubusercontent.com/sustainable-computing-io/kepler-model-db/main/models/v0.7/specpower/acpi/AbsPower/BPFOnly/GradientBoostingRegressorTrainer_0.zip
+    NODE_TOTAL_INIT_URL=https://raw.githubusercontent.com/sustainable-computing-io/kepler-model-db/main/models/v0.7/specpower-0.7.11/acpi/AbsPower/BPFOnly/SGDRegressorTrainer_0.zip
 ---
 apiVersion: apps/v1
 kind: DaemonSet

src/kepler_model/estimate/estimator.py

@@ -9,7 +9,7 @@
 
 import socket
 import signal
-from kepler_model.estimate.model_server_connector import make_request
+from kepler_model.estimate.model_server_connector import make_request, is_model_server_enabled
 from kepler_model.estimate.archived_model import get_achived_model
 from kepler_model.estimate.model.model import load_downloaded_model
 from kepler_model.util.loader import get_download_output_path
@@ -64,15 +64,19 @@ def handle_request(data):
     if output_type.name not in loaded_model:
         loaded_model[output_type.name] = dict()
     output_path = ""
-    request_trainer = False
-    if power_request.trainer_name is not None:
-        if output_type.name in loaded_model and power_request.energy_source in loaded_model[output_type.name]:
-            current_trainer = loaded_model[output_type.name][power_request.energy_source].trainer_name
-            request_trainer = current_trainer != power_request.trainer_name
-            if request_trainer:
-                logger.info(f"try obtaining the requesting trainer {power_request.trainer_name} (current: {current_trainer})")
-    if power_request.energy_source not in loaded_model[output_type.name] or request_trainer:
+    mismatch_trainer = False
+    if is_model_server_enabled():
+        if power_request.trainer_name is not None and power_request.trainer_name:
+            if output_type.name in loaded_model and power_request.energy_source in loaded_model[output_type.name]:
+                current_trainer = loaded_model[output_type.name][power_request.energy_source].trainer_name
+                mismatch_trainer = current_trainer != power_request.trainer_name
+                if mismatch_trainer:
+                    logger.info(f"try obtaining the requesting trainer {power_request.trainer_name} (current: {current_trainer})")
+    if power_request.energy_source not in loaded_model[output_type.name] or mismatch_trainer:
         output_path = get_download_output_path(download_path, power_request.energy_source, output_type)
+        if  mismatch_trainer and os.path.exists(output_path):
+            # remove existing model if mismatch
+            shutil.rmtree(output_path)
         if not os.path.exists(output_path):
             # try connecting to model server
             output_path = make_request(power_request)
@@ -87,12 +91,11 @@ def handle_request(data):
                     logger.info(f"load model from config: {output_path}")
             else:
                 logger.info(f"load model from model server: {output_path}")
+
         loaded_item = load_downloaded_model(power_request.energy_source, output_type)
         if loaded_item is not None and loaded_item.estimator is not None:
             loaded_model[output_type.name][power_request.energy_source] = loaded_item
             logger.info(f"set model {loaded_item.model_name} for {output_type.name} ({power_request.energy_source})")
-        # remove loaded model
-        shutil.rmtree(output_path)
 
     model = loaded_model[output_type.name][power_request.energy_source]
     powers, msg = model.get_power(power_request.datapoint)

src/kepler_model/estimate/model_server_connector.py

@@ -35,8 +35,9 @@ def unpack(energy_source, output_type, response, replace=True):
     tmp_filepath = os.path.join(download_path, TMP_FILE)
     if os.path.exists(output_path):
         if not replace:
-            # delete downloaded file
-            os.remove(tmp_filepath)
+            if os.path.exists(tmp_filepath):
+                # delete downloaded file
+                os.remove(tmp_filepath)
             return output_path
         # delete existing model
         shutil.rmtree(output_path)