Skip to content

Commit a9818df

Browse files
thieu1995thieu1995
committed
Updated all script runs
1 parent cf73d10 commit a9818df

File tree

10 files changed

+346
-366
lines changed

10 files changed

+346
-366
lines changed

‎abfo_mlnn_script.py‎

Lines changed: 37 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,58 +1,60 @@
1+
#!/usr/bin/env python
2+
# ------------------------------------------------------------------------------------------------------%
3+
# Created by "Thieu Nguyen" at 00:51, 29/03/2020 %
4+
# %
5+
# Email: [email protected] %
6+
# Homepage: https://www.researchgate.net/profile/Thieu_Nguyen6 %
7+
# Github: https://github.com/thieunguyen5991 %
8+
# -------------------------------------------------------------------------------------------------------%
9+
10+
from os.path import splitext, basename, realpath
111
from sklearn.model_selection import ParameterGrid
212
from model.main.hybrid_mlnn import ABfoLSMlnn
3-
from utils.IOUtil import read_dataset_file
413
from utils.SettingPaper import abfols_mlnn_paras_final as param_grid
5-
from utils.SettingPaper import ggtrace_cpu, ggtrace_ram, ggtrace_multi_cpu, ggtrace_multi_ram, traffic_eu, traffic_uk, worldcup
14+
from utils.SettingPaper import *
15+
from utils.IOUtil import load_dataset
616

7-
rv_data = [ggtrace_cpu, ggtrace_ram, ggtrace_multi_cpu, ggtrace_multi_ram, traffic_eu, traffic_uk, worldcup]
8-
data_file = ["google_5m", "google_5m", "google_5m", "google_5m", "it_eu_5m", "it_uk_5m", "worldcup98_5m"]
9-
test_type = "normal" ### normal: for normal test, stability: for n_times test
10-
run_times = None
17+
if SP_RUN_TIMES == 1:
18+
all_model_file_name = SP_LOG_FILENAME
19+
else: # If runs with more than 1, like stability test --> name of the models ==> such as: rnn1hl.csv
20+
all_model_file_name = str(splitext(basename(realpath(__file__)))[0])
1121

12-
if test_type == "normal": ### For normal test
13-
run_times = 1
14-
pathsave = "paper/results/final/"
15-
all_model_file_name = "ffnn_log_models"
16-
elif test_type == "stability": ### For stability test (n times run with the same parameters)
17-
run_times = 15
18-
pathsave = "paper/results/stability/"
19-
all_model_file_name = "stability_abfols"
20-
else:
21-
pass
2222

2323
def train_model(item):
2424
root_base_paras = {
2525
"dataset": dataset,
26-
"data_idx": (0.7, 0.15, 0.15),
26+
"feature_size": feature_size,
27+
"data_idx": SP_DATA_SPLIT_INDEX_2,
2728
"sliding": item["sliding"],
28-
"multi_output": requirement_variables[2],
29-
"output_idx": requirement_variables[3],
30-
"method_statistic": 0, # 0: sliding window, 1: mean, 2: min-mean-max, 3: min-median-max
29+
"multi_output": multi_output,
30+
"output_idx": output_index,
31+
"method_statistic": SP_PREPROCESSING_METHOD,
3132
"log_filename": all_model_file_name,
32-
"path_save_result": pathsave + requirement_variables[4],
33-
"test_type": test_type,
34-
"draw": True,
35-
"print_train": 1 # 0: nothing, 1 : full detail, 2: short version
33+
"n_runs": SP_RUN_TIMES, # 1 or others
34+
"path_save_result": SP_PATH_SAVE_BASE + SP_DATA_FILENAME[loop] + "/",
35+
"draw": SP_DRAW,
36+
"print_train": SP_PRINT_TRAIN, # 0: nothing, 1 : full detail, 2: short version
3637
}
38+
paras_name = "hs_{}-ep_{}-act_{}-Ci_{}-Ped_{}-Ns_{}-N_minmax_{}".format(item["hidden_size"], item["epoch"], item["activation"],
39+
item["Ci"], item["Ped"], item["Ns"], item["N_minmax"])
3740
root_hybrid_paras = {
38-
"hidden_size": item["hidden_size"], "activations": item["activations"], "epoch": item["epoch"],
39-
"train_valid_rate": item["train_valid_rate"], "domain_range": item["domain_range"]
41+
"hidden_size": item["hidden_size"], "activations": item["activations"], "epoch": item["epoch"], "domain_range": item["domain_range"],
42+
"paras_name": paras_name
4043
}
4144
abfols_paras = {
42-
"epoch": item["epoch"], "pop_size": item["pop_size"], "Ci": item["Ci"], "Ped": item["Ped"], "Ns": item["Ns"],
43-
"N_minmax": item["N_minmax"]
45+
"epoch": item["epoch"], "pop_size": item["pop_size"], "Ci": item["Ci"], "Ped": item["Ped"], "Ns": item["Ns"], "N_minmax": item["N_minmax"]
4446
}
4547
md = ABfoLSMlnn(root_base_paras=root_base_paras, root_hybrid_paras=root_hybrid_paras, abfols_paras=abfols_paras)
4648
md._running__()
4749

4850

49-
for _ in range(run_times):
50-
for loop in range(len(rv_data)):
51-
requirement_variables = rv_data[loop]
52-
filename = requirement_variables[0] + data_file[loop] + ".csv"
53-
dataset = read_dataset_file(filename, requirement_variables[1])
51+
for _ in range(SP_RUN_TIMES):
52+
for loop in range(len(SP_DATA_FILENAME)):
53+
filename = SP_LOAD_DATA_FROM + SP_DATA_FILENAME[loop]
54+
dataset = load_dataset(filename, cols=SP_DATA_COLS[loop])
55+
feature_size = len(SP_DATA_COLS[loop])
56+
multi_output = SP_DATA_MULTI_OUTPUT[loop]
57+
output_index = SP_OUTPUT_INDEX[loop]
5458
# Create combination of params.
5559
for item in list(ParameterGrid(param_grid)):
5660
train_model(item)
57-
58-

‎cro_mlnn_script.py‎

Lines changed: 36 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,43 +1,45 @@
1+
#!/usr/bin/env python
2+
# ------------------------------------------------------------------------------------------------------%
3+
# Created by "Thieu Nguyen" at 00:51, 29/03/2020 %
4+
# %
5+
# Email: [email protected] %
6+
# Homepage: https://www.researchgate.net/profile/Thieu_Nguyen6 %
7+
# Github: https://github.com/thieunguyen5991 %
8+
# -------------------------------------------------------------------------------------------------------%
9+
10+
from os.path import splitext, basename, realpath
111
from sklearn.model_selection import ParameterGrid
212
from model.main.hybrid_mlnn import CroMlnn
3-
from utils.IOUtil import read_dataset_file
413
from utils.SettingPaper import cro_mlnn_paras_final as param_grid
5-
from utils.SettingPaper import ggtrace_cpu, ggtrace_ram, ggtrace_multi_cpu, ggtrace_multi_ram, traffic_eu, traffic_uk, worldcup
6-
7-
rv_data = [ggtrace_cpu, ggtrace_ram, ggtrace_multi_cpu, ggtrace_multi_ram, traffic_eu, traffic_uk, worldcup]
8-
data_file = ["google_5m", "google_5m", "google_5m", "google_5m", "it_eu_5m", "it_uk_5m", "worldcup98_5m"]
9-
test_type = "normal" ### normal: for normal test, stability: for n_times test
10-
run_times = None
14+
from utils.SettingPaper import *
15+
from utils.IOUtil import load_dataset
1116

12-
if test_type == "normal": ### For normal test
13-
run_times = 1
14-
pathsave = "paper/results/final/"
15-
all_model_file_name = "ffnn_log_models"
16-
elif test_type == "stability": ### For stability test (n times run with the same parameters)
17-
run_times = 15
18-
pathsave = "paper/results/stability/mlnn/"
19-
all_model_file_name = "stability_cro_mlnn"
20-
else:
21-
pass
17+
if SP_RUN_TIMES == 1:
18+
all_model_file_name = SP_LOG_FILENAME
19+
else: # If runs with more than 1, like stability test --> name of the models ==> such as: rnn1hl.csv
20+
all_model_file_name = str(splitext(basename(realpath(__file__)))[0])
2221

2322

2423
def train_model(item):
2524
root_base_paras = {
2625
"dataset": dataset,
27-
"data_idx": (0.7, 0.15, 0.15),
26+
"feature_size": feature_size,
27+
"data_idx": SP_DATA_SPLIT_INDEX_2,
2828
"sliding": item["sliding"],
29-
"multi_output": requirement_variables[2],
30-
"output_idx": requirement_variables[3],
31-
"method_statistic": 0, # 0: sliding window, 1: mean, 2: min-mean-max, 3: min-median-max
29+
"multi_output": multi_output,
30+
"output_idx": output_index,
31+
"method_statistic": SP_PREPROCESSING_METHOD,
3232
"log_filename": all_model_file_name,
33-
"path_save_result": pathsave + requirement_variables[4],
34-
"test_type": test_type,
35-
"draw": True,
36-
"print_train": 1 # 0: nothing, 1 : full detail, 2: short version
33+
"n_runs": SP_RUN_TIMES, # 1 or others
34+
"path_save_result": SP_PATH_SAVE_BASE + SP_DATA_FILENAME[loop] + "/",
35+
"draw": SP_DRAW,
36+
"print_train": SP_PRINT_TRAIN, # 0: nothing, 1 : full detail, 2: short version
3737
}
38+
paras_name = "hs_{}-ep_{}-act_{}-po_{}-Fb_{}-Fa_{}-Fd_{}-Pd_{}-G_{}-GCR_{}-k_{}".format(item["hidden_size"], item["epoch"], item["activations"],
39+
item["po"], item["Fb"], item["Fa"], item["Fd"], item["Pd"], item["G"], item["GCR"], item["k"])
3840
root_hybrid_paras = {
39-
"hidden_size": item["hidden_size"], "activations": item["activations"], "epoch": item["epoch"],
40-
"train_valid_rate": item["train_valid_rate"], "domain_range": item["domain_range"]
41+
"hidden_size": item["hidden_size"], "activations": item["activations"], "epoch": item["epoch"], "domain_range": item["domain_range"],
42+
"paras_name": paras_name
4143
}
4244
cro_paras = {
4345
"epoch": item["epoch"], "pop_size": item["pop_size"], "po": item["po"], "Fb": item["Fb"], "Fa": item["Fa"],
@@ -47,11 +49,13 @@ def train_model(item):
4749
md._running__()
4850

4951

50-
for _ in range(run_times):
51-
for loop in range(len(rv_data)):
52-
requirement_variables = rv_data[loop]
53-
filename = requirement_variables[0] + data_file[loop] + ".csv"
54-
dataset = read_dataset_file(filename, requirement_variables[1])
52+
for _ in range(SP_RUN_TIMES):
53+
for loop in range(len(SP_DATA_FILENAME)):
54+
filename = SP_LOAD_DATA_FROM + SP_DATA_FILENAME[loop]
55+
dataset = load_dataset(filename, cols=SP_DATA_COLS[loop])
56+
feature_size = len(SP_DATA_COLS[loop])
57+
multi_output = SP_DATA_MULTI_OUTPUT[loop]
58+
output_index = SP_OUTPUT_INDEX[loop]
5559
# Create combination of params.
5660
for item in list(ParameterGrid(param_grid)):
5761
train_model(item)

‎ga_mlnn_script.py‎

Lines changed: 35 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,43 +1,44 @@
1+
#!/usr/bin/env python
2+
# ------------------------------------------------------------------------------------------------------%
3+
# Created by "Thieu Nguyen" at 00:51, 29/03/2020 %
4+
# %
5+
# Email: [email protected] %
6+
# Homepage: https://www.researchgate.net/profile/Thieu_Nguyen6 %
7+
# Github: https://github.com/thieunguyen5991 %
8+
# -------------------------------------------------------------------------------------------------------%
9+
10+
from os.path import splitext, basename, realpath
111
from sklearn.model_selection import ParameterGrid
212
from model.main.hybrid_mlnn import GaMlnn
3-
from utils.IOUtil import read_dataset_file
413
from utils.SettingPaper import ga_mlnn_paras_final as param_grid
5-
from utils.SettingPaper import ggtrace_cpu, ggtrace_ram, ggtrace_multi_cpu, ggtrace_multi_ram, traffic_eu, traffic_uk, worldcup
6-
7-
rv_data = [ggtrace_cpu, ggtrace_ram, ggtrace_multi_cpu, ggtrace_multi_ram, traffic_eu, traffic_uk, worldcup]
8-
data_file = ["google_5m", "google_5m", "google_5m", "google_5m", "it_eu_5m", "it_uk_5m", "worldcup98_5m"]
9-
test_type = "normal" ### normal: for normal test, stability: for n_times test
10-
run_times = None
14+
from utils.SettingPaper import *
15+
from utils.IOUtil import load_dataset
1116

12-
if test_type == "normal": ### For normal test
13-
run_times = 1
14-
pathsave = "paper/results/final/"
15-
all_model_file_name = "ffnn_log_models"
16-
elif test_type == "stability": ### For stability test (n times run with the same parameters)
17-
run_times = 15
18-
pathsave = "paper/results/stability/mlnn/"
19-
all_model_file_name = "stability_ga_mlnn"
20-
else:
21-
pass
17+
if SP_RUN_TIMES == 1:
18+
all_model_file_name = SP_LOG_FILENAME
19+
else: # If runs with more than 1, like stability test --> name of the models ==> such as: rnn1hl.csv
20+
all_model_file_name = str(splitext(basename(realpath(__file__)))[0])
2221

2322

2423
def train_model(item):
2524
root_base_paras = {
2625
"dataset": dataset,
27-
"data_idx": (0.7, 0.15, 0.15),
26+
"feature_size": feature_size,
27+
"data_idx": SP_DATA_SPLIT_INDEX_2,
2828
"sliding": item["sliding"],
29-
"multi_output": requirement_variables[2],
30-
"output_idx": requirement_variables[3],
31-
"method_statistic": 0, # 0: sliding window, 1: mean, 2: min-mean-max, 3: min-median-max
29+
"multi_output": multi_output,
30+
"output_idx": output_index,
31+
"method_statistic": SP_PREPROCESSING_METHOD,
3232
"log_filename": all_model_file_name,
33-
"path_save_result": pathsave + requirement_variables[4],
34-
"test_type": test_type,
35-
"draw": True,
36-
"print_train": 1 # 0: nothing, 1 : full detail, 2: short version
33+
"n_runs": SP_RUN_TIMES, # 1 or others
34+
"path_save_result": SP_PATH_SAVE_BASE + SP_DATA_FILENAME[loop] + "/",
35+
"draw": SP_DRAW,
36+
"print_train": SP_PRINT_TRAIN, # 0: nothing, 1 : full detail, 2: short version
3737
}
38+
paras_name = "hs_{}-ep_{}-act_{}-pc_{}-pm_{}".format(item["hidden_size"], item["epoch"], item["activations"], item["pc"], item["pm"])
3839
root_hybrid_paras = {
39-
"hidden_size": item["hidden_size"], "activations": item["activations"], "epoch": item["epoch"],
40-
"train_valid_rate": item["train_valid_rate"], "domain_range": item["domain_range"]
40+
"hidden_size": item["hidden_size"], "activations": item["activations"], "epoch": item["epoch"], "domain_range": item["domain_range"],
41+
"paras_name": paras_name
4142
}
4243
ga_paras = {
4344
"epoch": item["epoch"], "pop_size": item["pop_size"], "pc": item["pc"], "pm": item["pm"]
@@ -46,12 +47,13 @@ def train_model(item):
4647
md._running__()
4748

4849

49-
for _ in range(run_times):
50-
for loop in range(len(rv_data)):
51-
requirement_variables = rv_data[loop]
52-
filename = requirement_variables[0] + data_file[loop] + ".csv"
53-
dataset = read_dataset_file(filename, requirement_variables[1])
50+
for _ in range(SP_RUN_TIMES):
51+
for loop in range(len(SP_DATA_FILENAME)):
52+
filename = SP_LOAD_DATA_FROM + SP_DATA_FILENAME[loop]
53+
dataset = load_dataset(filename, cols=SP_DATA_COLS[loop])
54+
feature_size = len(SP_DATA_COLS[loop])
55+
multi_output = SP_DATA_MULTI_OUTPUT[loop]
56+
output_index = SP_OUTPUT_INDEX[loop]
5457
# Create combination of params.
5558
for item in list(ParameterGrid(param_grid)):
5659
train_model(item)
57-

‎lstm1hl_script.py‎

Lines changed: 36 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,53 +1,57 @@
1+
# !/usr/bin/env python
2+
# ------------------------------------------------------------------------------------------------------%
3+
# Created by "Thieu Nguyen" at 00:51, 29/03/2020 %
4+
# %
5+
# Email: [email protected] %
6+
# Homepage: https://www.researchgate.net/profile/Thieu_Nguyen6 %
7+
# Github: https://github.com/thieunguyen5991 %
8+
# -------------------------------------------------------------------------------------------------------%
9+
10+
from os.path import splitext, basename, realpath
111
from sklearn.model_selection import ParameterGrid
212
from model.main.traditional_rnn import Lstm1HL
3-
from utils.IOUtil import read_dataset_file
413
from utils.SettingPaper import lstm1hl_paras_final as param_grid
5-
from utils.SettingPaper import ggtrace_cpu, ggtrace_ram, ggtrace_multi_cpu, ggtrace_multi_ram, traffic_eu, traffic_uk, worldcup
14+
from utils.SettingPaper import *
15+
from utils.IOUtil import load_dataset
616

7-
rv_data = [ggtrace_cpu, ggtrace_ram, ggtrace_multi_cpu, ggtrace_multi_ram, traffic_eu, traffic_uk, worldcup]
8-
data_file = ["google_5m", "google_5m", "google_5m", "google_5m", "it_eu_5m", "it_uk_5m", "worldcup98_5m"]
9-
test_type = "normal" ### normal: for normal test, stability: for n_times test
10-
run_times = None
17+
if SP_RUN_TIMES == 1:
18+
all_model_file_name = SP_LOG_FILENAME
19+
else: # If runs with more than 1, like stability test --> name of the models ==> such as: rnn1hl.csv
20+
all_model_file_name = str(splitext(basename(realpath(__file__)))[0])
1121

12-
if test_type == "normal": ### For normal test
13-
run_times = 1
14-
pathsave = "paper/results/final/"
15-
all_model_file_name = "nn_log_models"
16-
elif test_type == "stability": ### For stability test (n times run with the same parameters)
17-
run_times = 15
18-
pathsave = "paper/results/stability/network/"
19-
all_model_file_name = "stability_lstm1hl"
20-
else:
21-
pass
2222

2323
def train_model(item):
2424
root_base_paras = {
2525
"dataset": dataset,
26-
"data_idx": (0.8, 0, 0.2),
26+
"feature_size": feature_size,
27+
"data_idx": SP_DATA_SPLIT_INDEX,
2728
"sliding": item["sliding"],
28-
"multi_output": requirement_variables[2],
29-
"output_idx": requirement_variables[3],
30-
"method_statistic": 0, # 0: sliding window, 1: mean, 2: min-mean-max, 3: min-median-max
29+
"multi_output": multi_output,
30+
"output_idx": output_index,
31+
"method_statistic": SP_PREPROCESSING_METHOD,
3132
"log_filename": all_model_file_name,
32-
"path_save_result": pathsave + requirement_variables[4],
33-
"test_type": test_type,
34-
"draw": True,
35-
"print_train": 0 # 0: nothing, 1 : full detail, 2: short version
33+
"n_runs": SP_RUN_TIMES, # 1 or others
34+
"path_save_result": SP_PATH_SAVE_BASE + SP_DATA_FILENAME[loop] + "/",
35+
"draw": SP_DRAW,
36+
"print_train": SP_PRINT_TRAIN, # 0: nothing, 1 : full detail, 2: short version
3637
}
38+
paras_name = "hs_{}-ep_{}-bs_{}-lr_{}-ac_{}-op_{}-lo_{}".format(item["hidden_sizes"], item["epoch"], item["batch_size"], item["learning_rate"],
39+
item["activations"], item["optimizer"], item["loss"])
3740
root_rnn_paras = {
38-
"hidden_sizes": item["hidden_sizes"], "epoch": item["epoch"], "batch_size": item["batch_size"],
39-
"learning_rate": item["learning_rate"], "activations": item["activations"],
40-
"optimizer": item["optimizer"], "loss": item["loss"], "dropouts": item["dropouts"]
41+
"hidden_sizes": item["hidden_sizes"], "epoch": item["epoch"], "batch_size": item["batch_size"], "learning_rate": item["learning_rate"],
42+
"activations": item["activations"], "optimizer": item["optimizer"], "loss": item["loss"], "dropouts": item["dropouts"], "paras_name": paras_name
4143
}
4244
md = Lstm1HL(root_base_paras=root_base_paras, root_rnn_paras=root_rnn_paras)
4345
md._running__()
4446

4547

46-
for _ in range(run_times):
47-
for loop in range(len(rv_data)):
48-
requirement_variables = rv_data[loop]
49-
filename = requirement_variables[0] + data_file[loop] + ".csv"
50-
dataset = read_dataset_file(filename, requirement_variables[1])
48+
for _ in range(SP_RUN_TIMES):
49+
for loop in range(len(SP_DATA_FILENAME)):
50+
filename = SP_LOAD_DATA_FROM + SP_DATA_FILENAME[loop]
51+
dataset = load_dataset(filename, cols=SP_DATA_COLS[loop])
52+
feature_size = len(SP_DATA_COLS[loop])
53+
multi_output = SP_DATA_MULTI_OUTPUT[loop]
54+
output_index = SP_OUTPUT_INDEX[loop]
5155
# Create combination of params.
5256
for item in list(ParameterGrid(param_grid)):
5357
train_model(item)

0 commit comments

Comments
 (0)