move download path to under mnt

Signed-off-by: Sunyanan Choochotkaew <[email protected]>

Author: sunya-ch

cmd/main.py

@@ -13,9 +13,6 @@
 UTC_OFFSET_TIMEDELTA = datetime.datetime.utcnow() - datetime.datetime.now()
 data_path = os.getenv("CPE_DATAPATH", data_path)
 
-# set model top path to data path
-os.environ['MODEL_PATH'] = data_path
-
 cur_path = os.path.join(os.path.dirname(__file__), '.')
 sys.path.append(cur_path)
 src_path = os.path.join(os.path.dirname(__file__), '..', 'src')
@@ -749,6 +746,9 @@ def export(args):
 
 
 if __name__ == "__main__":
+    # set model top path to data path
+    os.environ['MODEL_PATH'] = data_path
+
     # Create an ArgumentParser object
     parser = argparse.ArgumentParser(description="Kepler model server entrypoint")
     parser.add_argument("command", type=str, help="The command to execute.")

manifests/base/patch/patch-estimator-sidecar.yaml

@@ -37,6 +37,10 @@ spec:
           readOnly: true
         - mountPath: /tmp
           name: tmp
+        - mountPath: /mnt
+          name: mnt
       volumes:
       - emptyDir: {}
-        name: tmp
+        name: tmp
+      - emptyDir: {}
+        name: mnt

manifests/offline-trainer/offline-trainer.yaml

@@ -1,11 +1,3 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kepler-model-server-cfm
-  namespace: system
-data:
-  MODEL_PATH: /data/models
----
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -31,7 +23,7 @@ spec:
         configMap:
           name: kepler-model-server-cfm
       - emptyDir: {}
-        name: model-data
+        name: mnt
       containers:
       - name: offline-trainer
         image: kepler_model_server
@@ -43,8 +35,8 @@ spec:
           - name: cfm
             mountPath: /etc/kepler/kepler.config
             readOnly: true
-          - name: model-data
-            mountPath: /data
+          - name: mnt
+            mountPath: /mnt
             readOnly: false
         command: [ "python3.8" ]
         args: ["-u", "src/train/offline_trainer.py" ]

manifests/server/online-train/patch-trainer.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: kepler-model-server-cfm
-  namespace: system
+  namespace: kepler
 data:
   PROM_SERVER: 'http://prometheus-k8s.monitoring.svc.cluster.local:9090'
   PROM_QUERY_INTERVAL: '20'
@@ -14,7 +14,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: kepler-model-server
-  namespace: system
+  namespace: kepler
 spec:
   template:
     spec:
@@ -27,8 +27,8 @@ spec:
           - name: cfm
             mountPath: /etc/kepler/kepler.config
             readOnly: true
-          - name: model-data
-            mountPath: /data
+          - name: mnt
+            mountPath: /mnt
             readOnly: false
         command: [ "python3.8" ]
         args: ["-u", "src/train/online_trainer.py" ]

manifests/server/server.yaml

@@ -3,8 +3,6 @@ kind: ConfigMap
 metadata:
   name: kepler-model-server-cfm
   namespace: system
-data:
-  MODEL_PATH: /data/models
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -31,7 +29,7 @@ spec:
         configMap:
           name: kepler-model-server-cfm
       - emptyDir: {}
-        name: model-data
+        name: mnt
       containers:
       - name: server-api
         image: kepler_model_server
@@ -43,8 +41,8 @@ spec:
           - name: cfm
             mountPath: /etc/kepler/kepler.config
             readOnly: true
-          - name: model-data
-            mountPath: /data
+          - name: mnt
+            mountPath: /mnt
             readOnly: false
         command: [ "python3.8" ]
         args: ["-u", "src/server/model_server.py" ]

src/estimate/estimator.py

@@ -37,7 +37,7 @@ def __init__(self, metrics, values, output_type, source, system_features, system
 from archived_model import get_achived_model
 from model import load_downloaded_model
 from loader import get_download_output_path
-from config import set_env_from_model_config, SERVE_SOCKET
+from config import set_env_from_model_config, SERVE_SOCKET, download_path
 from train_types import is_support_output_type
 
 loaded_model = dict()
@@ -56,7 +56,7 @@ def handle_request(data):
     output_type = ModelOutputType[power_request.output_type]
 
     if output_type.name not in loaded_model:
-        output_path = get_download_output_path(power_request.energy_source, output_type)
+        output_path = get_download_output_path(download_path, power_request.energy_source, output_type)
         if not os.path.exists(output_path):
             # try connecting to model server
             output_path = make_request(power_request)

src/estimate/model/model.py

@@ -11,6 +11,7 @@
 sys.path.append(cur_path)
 
 from loader import load_metadata, get_download_output_path
+from config import download_path
 from prom_types import TIMESTAMP_COL, valid_container_query
 
 from scikit_model import ScikitModel
@@ -132,5 +133,5 @@ def load_model(model_path):
 
 # download model folder has no subfolder of energy source and feature group because it has been already determined by model request
 def load_downloaded_model(energy_source, output_type):
-    model_path = get_download_output_path(energy_source, output_type)
+    model_path = get_download_output_path(download_path, energy_source, output_type)
     return load_model(model_path)

src/estimate/model_server_connector.py

@@ -9,8 +9,8 @@
 util_path = os.path.join(os.path.dirname(__file__), '..', 'util')
 sys.path.append(util_path)
 
-from config import is_model_server_enabled, get_model_server_req_endpoint, get_model_server_list_endpoint
-from loader import get_download_path, get_download_output_path
+from config import is_model_server_enabled, get_model_server_req_endpoint, get_model_server_list_endpoint, download_path
+from loader import get_download_output_path
 from train_types import ModelOutputType
 
 def make_model_request(power_request):
@@ -19,8 +19,8 @@ def make_model_request(power_request):
 TMP_FILE = 'tmp.zip'
 
 def unpack(energy_source, output_type, response, replace=True):
-    output_path = get_download_output_path(energy_source, output_type)
-    tmp_filepath = os.path.join(get_download_path(), TMP_FILE)
+    output_path = get_download_output_path(download_path, energy_source, output_type)
+    tmp_filepath = os.path.join(download_path, TMP_FILE)
     if os.path.exists(output_path):
         if not replace:
             # delete downloaded file

src/server/model_server.py

@@ -13,7 +13,7 @@
 sys.path.append(util_path)
 
 from util.train_types import get_valid_feature_groups, ModelOutputType, FeatureGroups, FeatureGroup
-from util.config import getConfig, model_toppath, ERROR_KEY, MODEL_SERVER_MODEL_REQ_PATH, MODEL_SERVER_MODEL_LIST_PATH, initial_pipeline_url
+from util.config import getConfig, model_toppath, ERROR_KEY, MODEL_SERVER_MODEL_REQ_PATH, MODEL_SERVER_MODEL_LIST_PATH, initial_pipeline_url, download_path
 from util.loader import parse_filters, is_valid_model, load_json, load_weight, get_model_group_path, get_archived_file, METADATA_FILENAME, CHECKPOINT_FOLDERNAME, get_pipeline_path, any_node_type, is_matched_type
 
 ###############################################
@@ -179,15 +179,16 @@ def load_init_pipeline():
     # unpack pipeline
     try:
         TMP_FILE = 'tmp.zip'
-        with codecs.open(TMP_FILE, 'wb') as f:
+        tmp_filepath = os.path.join(download_path, TMP_FILE)
+        with codecs.open(tmp_filepath, 'wb') as f:
             f.write(response.content)
-        shutil.unpack_archive(TMP_FILE, default_pipeline)
-    except:
-        print("failed to unpack downloaded pipeline.")
+        shutil.unpack_archive(tmp_filepath, default_pipeline)
+    except Exception as e:
+        print("failed to unpack downloaded pipeline: ", e)
         return
 
     # remove downloaded zip
-    os.remove(TMP_FILE)
+    os.remove(tmp_filepath)
     print("initial pipeline is loaded to {}".format(default_pipeline))
 
 if __name__ == '__main__':

src/util/config.py

@@ -21,6 +21,9 @@
 # can be read only (for configmap mount)
 CONFIG_PATH = "/etc/kepler/kepler.config"
 
+DOWNLOAD_FOLDERNAME = "download"
+MODEL_FOLDERNAME = "models"
+
 DEFAULT_TOTAL_SOURCE = "acpi"
 DEFAULT_COMPONENTS_SOURCE = "rapl"
 TOTAL_KEY = "TOTAL"
@@ -62,11 +65,8 @@ def getPath(subpath):
 
 initial_pipeline_url = getConfig('INITIAL_PIPELINE_URL', get_pipeline_url())
 
-default_model_path = os.path.join(os.path.dirname(__file__), '..', 'models')
-model_toppath =  getPath(getConfig('MODEL_PATH', default_model_path))
-
-if not os.path.exists(model_toppath):
-    os.mkdir(model_toppath)
+model_toppath =  getConfig('MODEL_PATH', getPath(MODEL_FOLDERNAME))
+download_path = getConfig('MODEL_PATH', getPath(DOWNLOAD_FOLDERNAME))
 
 ERROR_KEY = 'mae'
 ERROR_KEY = getConfig('ERROR_KEY', ERROR_KEY)