Merge pull request #232 from sunya-ch/spec-trainer

update curvefit (log func) and generate_spec

Author: rootfs

.github/workflows/push-pr.yml

@@ -153,7 +153,7 @@ jobs:
           echo "change=true" >> "$GITHUB_OUTPUT"
 
   tekton-test:
-    needs: [check-secret, check-branch, base-image]
+    needs: [check-secret, check-branch, check-change, base-image]
     if: always()
     uses: ./.github/workflows/tekton-test.yml
     with:
@@ -164,7 +164,7 @@ jobs:
       pipeline_name: std_v0.7
 
   integration-test:
-    needs: [check-secret, check-branch, base-image]
+    needs: [check-secret, check-branch, check-change, base-image]
     if: always()
     uses: ./.github/workflows/integration-test.yml
     with:

dockerfiles/requirements.txt

@@ -12,4 +12,5 @@ xgboost==2.0.1
 scikit-learn==1.1.2
 py-cpuinfo==9.0.0
 seaborn==0.12.2
-psutil==5.9.8
+psutil==5.9.8
+pyudev==0.24.1

src/train/profiler/node_type_index.py

@@ -33,27 +33,44 @@ def rename(name):
     return name
 
 def format_processor(processor):
-    return "_".join(re.sub(r'\(.*\)', '', rename(processor)).split()).replace("-", "_").replace("V", "v").replace("_v", "v")
+    return "_".join(re.sub(r'\(.*\)', '', rename(processor)).split()).replace("-", "_").lower().replace("_v", "v")
+
+def format_vendor(vendor):
+    return vendor.split()[0].replace("-","_").replace(",","").replace("'","").lower()
 
 GB = 1024*1024*1024
 import psutil
 import cpuinfo
+import subprocess
+
+import pyudev
+
 def generate_spec(data_path, machine_id):
-    processor = "unknown"
+    processor = ""
+    vendor = ""
     cpu_info = cpuinfo.get_cpu_info()
     if "brand_raw" in cpu_info:
         processor = format_processor(cpu_info["brand_raw"])
+    context = pyudev.Context()
+    for device in context.list_devices(subsystem="dmi"):
+        if device.get('ID_VENDOR') is not None:
+            vendor = format_vendor(device.get('ID_VENDOR'))
+            break
     cores = psutil.cpu_count(logical=True)
-    chips = int(cores/psutil.cpu_count(logical=False))
+    chips = max(1, int(subprocess.check_output('cat /proc/cpuinfo | grep "physical id" | sort -u | wc -l', shell=True)))
+    threads_per_core = max(1, cores//psutil.cpu_count(logical=False))
     memory = psutil.virtual_memory().total
     memory_gb = int(memory/GB)
-    cpu_freq_mhz = round(psutil.cpu_freq(percpu=False).max/100)*100 # round to one decimal of GHz
+    freq = psutil.cpu_freq(percpu=False)
+    cpu_freq_mhz = round(max(freq.max, freq.current)/100)*100 # round to one decimal of GHz
     spec_values = {
+        "vendor": vendor,
         "processor": processor,
         "cores": cores,
         "chips": chips,
         "memory_gb": memory_gb,
-        "cpu_freq_mhz": cpu_freq_mhz
+        "cpu_freq_mhz": cpu_freq_mhz,
+        "threads_per_core": threads_per_core
     }
     spec = NodeTypeSpec(**spec_values)
     print("Save machine spec ({}): ".format(data_path))

src/train/trainer/ExponentialRegressionTrainer/main.py

@@ -6,6 +6,13 @@
 from trainer.curvefit import CurveFitTrainer, CurveFitModel
 
 import numpy as np
+import math
+
+def p0_func(x, y):
+    a = (y.max()-y.min())//math.e # scale value
+    b = 1 # start from linear
+    c = y.min() - a # initial offset
+    return [a,b,c]
 
 def expo_func(x, a, b, c):
     y = a*np.exp(b*x) + c 
@@ -18,4 +25,4 @@ def __init__(self, energy_components, feature_group, energy_source, node_level,
         self.fe_files = []
 
     def init_model(self):
-        return CurveFitModel(expo_func)
+        return CurveFitModel(expo_func, p0_func=p0_func)

src/train/trainer/LogarithmicRegressionTrainer/main.py

@@ -8,13 +8,13 @@
 import numpy as np
 
 def p0_func(x, y):
-    print(y.max(), y.min())
     a = y.max()-y.min()
-    b = y.min()
-    return [a, b]
+    b = 1
+    c = y.min()
+    return [a, b, c]
 
-def log_func(x, a, b):
-    y = [a * np.log(xi) + b if xi > 0 else 0 for xi in x]
+def log_func(x, a, b, c):
+    y = a*np.log(b*x+1) + c
     return y
 
 class LogarithmicRegressionTrainer(CurveFitTrainer):

src/train/trainer/LogisticRegressionTrainer/main.py

@@ -10,7 +10,7 @@
 def p0_func(x, y):
     A = y.max() - y.min() # value range
     x0 = 0.5 # sigmoid mid point (as normalized value is in 0 to 1, start mid point = 0.5)
-    k = A/np.std(y) # growth rate (larger std, lower growth)
+    k = A//np.std(y) # growth rate (larger std, lower growth)
     off = y.min() # initial offset
     return [A,x0,k,off]
 

src/train/trainer/curvefit.py

@@ -36,9 +36,9 @@ def fit(self, X_values, y_values):
         flatten_x = self._x_values(X_values)
         flatten_y = np.array(y_values).flatten()
         if self.p0_func is not None:
-            self.popt, self.pcov = curve_fit(self.fit_func, flatten_x, flatten_y, p0=self.p0_func(flatten_x, flatten_y), maxfev=20000)
+            self.popt, self.pcov = curve_fit(self.fit_func, flatten_x, flatten_y, p0=self.p0_func(flatten_x, flatten_y), maxfev=30000)
         else:
-            self.popt, self.pcov = curve_fit(self.fit_func, flatten_x, flatten_y, maxfev=20000)
+            self.popt, self.pcov = curve_fit(self.fit_func, flatten_x, flatten_y, maxfev=30000)
 
     def predict(self, X_values):
         if self.popt is None: