Merge pull request #158 from sunya-ch/setup-manifest

sunya-ch · web-flow · commit aa80bdc5cc31 · 2023-09-13T13:50:43.000+09:00
add missing cases of openshift deployment
diff --git a/Makefile b/Makefile
@@ -72,21 +72,31 @@ test: build-test test-pipeline test-estimator test-model-server test-offline-tra
 
 # set image
 set-image:
-	cd ./manifests/base && kustomize edit set image kepler_model_server=$(IMAGE)
-	cd ./manifests/server && kustomize edit set image kepler_model_server=$(IMAGE)
+	@cd ./manifests/base && kustomize edit set image kepler_model_server=$(IMAGE)
+	@cd ./manifests/server && kustomize edit set image kepler_model_server=$(IMAGE)
 
 # deploy
 _deploy:
-	$(MAKE) set-image
-	kustomize build ./manifests/base|kubectl apply -f -
+	@$(MAKE) set-image
+	@kustomize build ./manifests/base|kubectl apply -f -
+
+# print
+_print:
+	@$(MAKE) set-image
+	@kustomize build ./manifests/base|cat
 
 cleanup:
 	kustomize build manifests/base|kubectl delete -f -
 
 deploy:
-	chmod +x ./manifests/set.sh
-	./manifests/set.sh "${OPTS}"
-	$(MAKE) _deploy
+	@chmod +x ./manifests/set.sh
+	@./manifests/set.sh "${OPTS}"
+	@$(MAKE) _deploy
+
+manifest: 
+	@chmod +x ./manifests/set.sh
+	@./manifests/set.sh "${OPTS}"
+	@$(MAKE) _print
 
 e2e-test:
 	chmod +x ./tests/e2e_test.sh
diff --git a/cmd/main.py b/cmd/main.py
@@ -13,9 +13,6 @@
 UTC_OFFSET_TIMEDELTA = datetime.datetime.utcnow() - datetime.datetime.now()
 data_path = os.getenv("CPE_DATAPATH", data_path)
 
-# set model top path to data path
-os.environ['MODEL_PATH'] = data_path
-
 cur_path = os.path.join(os.path.dirname(__file__), '.')
 sys.path.append(cur_path)
 src_path = os.path.join(os.path.dirname(__file__), '..', 'src')
@@ -749,6 +746,9 @@ def export(args):
             
 
 if __name__ == "__main__":
+    # set model top path to data path
+    os.environ['MODEL_PATH'] = data_path
+
     # Create an ArgumentParser object
     parser = argparse.ArgumentParser(description="Kepler model server entrypoint")
     parser.add_argument("command", type=str, help="The command to execute.")
diff --git a/manifests/base/openshift/estimate-only/kustomization.yaml b/manifests/base/openshift/estimate-only/kustomization.yaml
@@ -0,0 +1,9 @@
+namespace: kepler
+
+patchesStrategicMerge:
+- ./patch/patch-estimator-sidecar.yaml
+- ./patch/patch-openshift.yaml
+
+resources:
+- ../kepler
+- ./openshift/scc.yaml
diff --git a/manifests/base/openshift/estimate-with-server/kustomization.yaml b/manifests/base/openshift/estimate-with-server/kustomization.yaml
@@ -0,0 +1,11 @@
+namespace: kepler
+
+patchesStrategicMerge:
+- ./patch/patch-estimator-sidecar.yaml
+- ./patch/patch-model-server.yaml
+- ./patch/patch-openshift.yaml
+
+resources:
+- ../kepler
+- ../server
+- ./openshift/scc.yaml
diff --git a/manifests/base/openshift/scc.yaml b/manifests/base/openshift/scc.yaml
@@ -0,0 +1,30 @@
+# scc for the Kepler
+kind: SecurityContextConstraints
+apiVersion: security.openshift.io/v1
+metadata:
+  name: kepler-scc
+# To allow running privilegedContainers
+allowPrivilegedContainer: true
+allowHostDirVolumePlugin: true
+allowHostNetwork: false
+allowHostPorts: false
+allowHostIPC: false
+allowHostPID: true
+readOnlyRootFilesystem: true
+defaultAddCapabilities:
+  - SYS_ADMIN
+runAsUser:
+  type: RunAsAny
+seLinuxContext:
+  type: RunAsAny
+fsGroup:
+  type: RunAsAny
+volumes:
+  - configMap
+  - projected
+  - emptyDir
+  - hostPath
+  - secret
+users:
+  - kepler
+  - system:serviceaccount:kepler:kepler-sa
diff --git a/manifests/base/openshift/serve-only/kustomization.yaml b/manifests/base/openshift/serve-only/kustomization.yaml
@@ -0,0 +1,10 @@
+namespace: kepler
+
+patchesStrategicMerge:
+- ./patch/patch-model-server.yaml
+- ./patch/patch-openshift.yaml
+
+resources:
+- ../kepler
+- ../server
+- ./openshift/scc.yaml
diff --git a/manifests/base/patch/patch-estimator-sidecar.yaml b/manifests/base/patch/patch-estimator-sidecar.yaml
@@ -37,6 +37,10 @@ spec:
           readOnly: true
         - mountPath: /tmp
           name: tmp
+        - mountPath: /mnt
+          name: mnt
       volumes:
       - emptyDir: {}
-        name: tmp
+        name: tmp
+      - emptyDir: {}
+        name: mnt
diff --git a/manifests/base/patch/patch-openshift.yaml b/manifests/base/patch/patch-openshift.yaml
@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  annotations:
+    openshift.io/description: "Kepler exporter"
+    openshift.io/display-name: ""
+  name: kepler
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kepler-exporter
+  namespace: kepler
+spec:
+  template:
+    spec:
+      containers:
+      - name: kepler-exporter
+        volumeMounts:
+        - name: kernel-src
+          mountPath: /usr/src/kernels
+        securityContext:
+          privileged: true
+      volumes:      
+      - name: kernel-src
+        hostPath:
+          path: /usr/src/kernels
+          type: Directory
diff --git a/manifests/offline-trainer/offline-trainer.yaml b/manifests/offline-trainer/offline-trainer.yaml
@@ -1,11 +1,3 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kepler-model-server-cfm
-  namespace: system
-data:
-  MODEL_PATH: /data/models
----
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -31,7 +23,7 @@ spec:
         configMap:
           name: kepler-model-server-cfm
       - emptyDir: {}
-        name: model-data
+        name: mnt
       containers:
       - name: offline-trainer
         image: kepler_model_server
@@ -43,8 +35,8 @@ spec:
           - name: cfm
             mountPath: /etc/kepler/kepler.config
             readOnly: true
-          - name: model-data
-            mountPath: /data
+          - name: mnt
+            mountPath: /mnt
             readOnly: false
         command: [ "python3.8" ]
         args: ["-u", "src/train/offline_trainer.py" ]
diff --git a/manifests/server/online-train/patch-trainer.yaml b/manifests/server/online-train/patch-trainer.yaml
@@ -2,7 +2,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: kepler-model-server-cfm
-  namespace: system
+  namespace: kepler
 data:
   PROM_SERVER: 'http://prometheus-k8s.monitoring.svc.cluster.local:9090'
   PROM_QUERY_INTERVAL: '20'
@@ -14,7 +14,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: kepler-model-server
-  namespace: system
+  namespace: kepler
 spec:
   template:
     spec:
@@ -27,8 +27,8 @@ spec:
           - name: cfm
             mountPath: /etc/kepler/kepler.config
             readOnly: true
-          - name: model-data
-            mountPath: /data
+          - name: mnt
+            mountPath: /mnt
             readOnly: false
         command: [ "python3.8" ]
         args: ["-u", "src/train/online_trainer.py" ]
diff --git a/manifests/server/server.yaml b/manifests/server/server.yaml
@@ -3,8 +3,6 @@ kind: ConfigMap
 metadata:
   name: kepler-model-server-cfm
   namespace: system
-data:
-  MODEL_PATH: /data/models
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -31,7 +29,7 @@ spec:
         configMap:
           name: kepler-model-server-cfm
       - emptyDir: {}
-        name: model-data
+        name: mnt
       containers:
       - name: server-api
         image: kepler_model_server
@@ -43,8 +41,8 @@ spec:
           - name: cfm
             mountPath: /etc/kepler/kepler.config
             readOnly: true
-          - name: model-data
-            mountPath: /data
+          - name: mnt
+            mountPath: /mnt
             readOnly: false
         command: [ "python3.8" ]
         args: ["-u", "src/server/model_server.py" ]
diff --git a/manifests/set.sh b/manifests/set.sh
@@ -33,35 +33,53 @@ fi
 
 echo "Preparing manifests..."
 
-if [ ! -z ${ESTIMATOR} ]; then
-    echo "add estimator-sidecar"
-    cp ./manifests/base/estimate-only/kustomization.yaml ./manifests/base/kustomization.yaml
-fi
-
 if [ ! -z ${SERVER} ]; then
     echo "deploy model server"
     if [ ! -z ${ESTIMATOR} ]; then
-        # with estimator, replace estimate-only with estimate-with-server
+        echo "add estimator-sidecar"
+        # OPTS="ESTIMATOR SERVER" --> base
         cp ./manifests/base/estimate-with-server/kustomization.yaml ./manifests/base/kustomization.yaml
+        if [ ! -z ${OPENSHIFT_DEPLOY} ]; then
+            echo "patch openshift deployment for exporter (estimator-with-server)"
+            # OPTS="ESTIMATOR SERVER OPENSHIFT_DEPLOY" --> base
+            cp ./manifests/base/openshift/estimate-with-server/kustomization.yaml ./manifests/base/kustomization.yaml
+        fi
     else
+        # OPTS="SERVER" --> base
         cp ./manifests/base/serve-only/kustomization.yaml ./manifests/base/kustomization.yaml
+        if [ ! -z ${OPENSHIFT_DEPLOY} ]; then
+            echo "patch openshift deployment for exporter (serve-only)"
+            # OPTS="SERVER OPENSHIFT_DEPLOY" --> base
+            cp ./manifests/base/openshift/serve-only/kustomization.yaml ./manifests/base/kustomization.yaml
+        fi
     fi
-    # default
-    cp ./manifests/server/base/kustomization.yaml ./manifests/server/kustomization.yaml
-    if [ ! -z ${OPENSHIFT_DEPLOY} ]; then
-        # replace with openshift serve-only
-        echo "openshift deployment"
-        cp ./manifests/server/openshift/serve-only/kustomization.yaml ./manifests/server/kustomization.yaml
-    fi
-
+    
     if [ ! -z ${ONLINE_TRAINER} ]; then
-        # replace with online-train
         echo "add online trainer"
-        if [ ! -z ${OPENSHIFT_DEPLOY} ]; then 
-            cp ./manifests/server/openshift/online-train/kustomization.yaml ./manifests/server/kustomization.yaml
-        else
-            cp ./manifests/server/online-train/kustomization.yaml ./manifests/server/kustomization.yaml
+        # OPTS="... SERVER ONLINE_TRAINER" --> server
+        cp ./manifests/server/online-train/kustomization.yaml ./manifests/server/kustomization.yaml
+        if [ ! -z ${OPENSHIFT_DEPLOY} ]; then
+            echo "patch openshift deployment for server (with online trainer)"
+            # OPTS="... SERVER ONLINE_TRAINER OPENSHIFT_DEPLOY" --> server
+            cp ./manifests/server/openshift/online-train/kustomization.yaml ./manifests/server/kustomization.yaml   
         fi
+    else 
+        # OPTS="... SERVER" --> server
+        cp ./manifests/server/base/kustomization.yaml ./manifests/server/kustomization.yaml
+        if [ ! -z ${OPENSHIFT_DEPLOY} ]; then
+            echo "patch openshift deployment for server"
+            # OPTS="... SERVER OPENSHIFT_DEPLOY" --> server
+            cp ./manifests/server/openshift/serve-only/kustomization.yaml ./manifests/server/kustomization.yaml
+        fi
+    fi
+elif [ ! -z ${ESTIMATOR} ]; then
+    echo "add estimator-sidecar"
+    # OPTS="ESTIMATOR" --> base
+    cp ./manifests/base/estimate-only/kustomization.yaml ./manifests/base/kustomization.yaml
+    if [ ! -z ${OPENSHIFT_DEPLOY} ]; then
+        echo "patch openshift deployment for exporter (estimator-only)"
+        # OPTS="ESTIMATOR OPENSHIFT_DEPLOY" --> base
+        cp ./manifests/base/openshift/estimate-only/kustomization.yaml ./manifests/base/kustomization.yaml
     fi
 fi
 
diff --git a/src/estimate/estimator.py b/src/estimate/estimator.py
@@ -37,7 +37,7 @@ def __init__(self, metrics, values, output_type, source, system_features, system
 from archived_model import get_achived_model
 from model import load_downloaded_model
 from loader import get_download_output_path
-from config import set_env_from_model_config, SERVE_SOCKET
+from config import set_env_from_model_config, SERVE_SOCKET, download_path
 from train_types import is_support_output_type
 
 loaded_model = dict()
@@ -56,7 +56,7 @@ def handle_request(data):
     output_type = ModelOutputType[power_request.output_type]
 
     if output_type.name not in loaded_model:
-        output_path = get_download_output_path(power_request.energy_source, output_type)
+        output_path = get_download_output_path(download_path, power_request.energy_source, output_type)
         if not os.path.exists(output_path):
             # try connecting to model server
             output_path = make_request(power_request)
diff --git a/src/estimate/model/model.py b/src/estimate/model/model.py
@@ -11,6 +11,7 @@
 sys.path.append(cur_path)
 
 from loader import load_metadata, get_download_output_path
+from config import download_path
 from prom_types import TIMESTAMP_COL, valid_container_query
 
 from scikit_model import ScikitModel
@@ -132,5 +133,5 @@ def load_model(model_path):
 
 # download model folder has no subfolder of energy source and feature group because it has been already determined by model request
 def load_downloaded_model(energy_source, output_type):
-    model_path = get_download_output_path(energy_source, output_type)
+    model_path = get_download_output_path(download_path, energy_source, output_type)
     return load_model(model_path)
diff --git a/src/estimate/model_server_connector.py b/src/estimate/model_server_connector.py
@@ -9,8 +9,8 @@
 util_path = os.path.join(os.path.dirname(__file__), '..', 'util')
 sys.path.append(util_path)
 
-from config import is_model_server_enabled, get_model_server_req_endpoint, get_model_server_list_endpoint
-from loader import get_download_path, get_download_output_path
+from config import is_model_server_enabled, get_model_server_req_endpoint, get_model_server_list_endpoint, download_path
+from loader import get_download_output_path
 from train_types import ModelOutputType
 
 def make_model_request(power_request):
@@ -19,8 +19,8 @@ def make_model_request(power_request):
 TMP_FILE = 'tmp.zip'
 
 def unpack(energy_source, output_type, response, replace=True):
-    output_path = get_download_output_path(energy_source, output_type)
-    tmp_filepath = os.path.join(get_download_path(), TMP_FILE)
+    output_path = get_download_output_path(download_path, energy_source, output_type)
+    tmp_filepath = os.path.join(download_path, TMP_FILE)
     if os.path.exists(output_path):
         if not replace:
             # delete downloaded file
diff --git a/src/server/model_server.py b/src/server/model_server.py
diff --git a/src/util/config.py b/src/util/config.py
diff --git a/src/util/loader.py b/src/util/loader.py
diff --git a/tests/README.md b/tests/README.md
diff --git a/tests/estimator_model_request_test.py b/tests/estimator_model_request_test.py
diff --git a/tests/model_server_test.py b/tests/model_server_test.py
diff --git a/tests/weight_model_request_test.py b/tests/weight_model_request_test.py
