From 69406bd37a05b526c60de417c7426b7dae30c22f Mon Sep 17 00:00:00 2001 From: ayushi-3536 Date: Sun, 12 Mar 2023 22:23:35 +0100 Subject: [PATCH 01/17] mo_tabular --- .../workflows/run_singularity_versions.yml | 0 .github/workflows/run_tests.yml | 0 .gitignore | 0 LICENSE | 0 README.md | 0 changelog.md | 0 ci_scripts/codestyle.sh | 0 ci_scripts/container_examples.sh | 0 ci_scripts/install.sh | 0 ci_scripts/install_singularity.sh | 0 ci_scripts/local_examples.sh | 0 ci_scripts/script.sh | 0 codecov.yml | 0 .../create_connection_to_benchmark.py | 0 .../container/tabular_benchmark_example.py | 0 examples/container/xgboost_with_container.py | 0 examples/jahs_bench_201.json | 0 examples/local/xgboost_local.py | 0 .../SVMSurrogate_minicomparison.py | 0 examples/w_optimizer/cartpole_bohb.py | 0 examples/w_optimizer/cartpole_smachb.py | 0 extra_requirements/cartpole.json | 0 extra_requirements/examples.json | 0 extra_requirements/ml_mfbb.json | 0 extra_requirements/mo_cnn.json | 0 extra_requirements/multi_objective.json | 0 extra_requirements/nasbench_101.json | 0 extra_requirements/nasbench_1shot1.json | 0 extra_requirements/outlier_detection.json | 0 extra_requirements/paramnet.json | 0 extra_requirements/pybnn.json | 0 extra_requirements/svm.json | 0 extra_requirements/tabular_benchmarks.json | 0 extra_requirements/tests.json | 0 extra_requirements/xgboost.json | 0 extra_requirements/yahpo_gym.json | 0 hpobench/__init__.py | 0 hpobench/__version__.py | 0 hpobench/abstract_benchmark.py | 1 + hpobench/benchmarks/__init__.py | 0 hpobench/benchmarks/ml/README.md | 0 hpobench/benchmarks/ml/__init__.py | 24 +-- hpobench/benchmarks/ml/histgb_benchmark.py | 23 ++- hpobench/benchmarks/ml/lr_benchmark.py | 40 ++++- hpobench/benchmarks/ml/nn_benchmark.py | 34 +++- hpobench/benchmarks/ml/pybnn.py | 0 hpobench/benchmarks/ml/rf_benchmark.py | 37 +++- hpobench/benchmarks/ml/svm_benchmark.py | 37 +++- hpobench/benchmarks/ml/svm_benchmark_old.py | 0 hpobench/benchmarks/ml/tabular_benchmark.py | 0 hpobench/benchmarks/ml/xgboost_benchmark.py | 35 +++- .../benchmarks/ml/xgboost_benchmark_old.py | 0 hpobench/benchmarks/mo/__init__.py | 0 hpobench/benchmarks/mo/adult_benchmark.py | 0 hpobench/benchmarks/mo/cnn_benchmark.py | 0 hpobench/benchmarks/nas/__init__.py | 0 hpobench/benchmarks/nas/jahs_benchmarks.py | 0 hpobench/benchmarks/nas/nasbench_101.py | 0 hpobench/benchmarks/nas/nasbench_1shot1.py | 0 hpobench/benchmarks/nas/nasbench_201.py | 0 hpobench/benchmarks/nas/tabular_benchmarks.py | 0 hpobench/benchmarks/od/__init__.py | 0 hpobench/benchmarks/od/od_ae.py | 0 hpobench/benchmarks/od/od_benchmarks.py | 0 hpobench/benchmarks/od/od_kde.py | 0 hpobench/benchmarks/od/od_ocsvm.py | 0 hpobench/benchmarks/rl/__init__.py | 0 hpobench/benchmarks/rl/cartpole.py | 0 hpobench/benchmarks/rl/learna_benchmark.py | 0 hpobench/benchmarks/surrogates/__init__.py | 0 .../surrogates/paramnet_benchmark.py | 0 .../benchmarks/surrogates/svm_benchmark.py | 0 hpobench/benchmarks/surrogates/yahpo_gym.py | 0 hpobench/config.py | 0 hpobench/container/__init__.py | 0 hpobench/container/benchmarks/__init__.py | 0 hpobench/container/benchmarks/ml/__init__.py | 0 .../benchmarks/ml/histgb_benchmark.py | 8 +- .../container/benchmarks/ml/lr_benchmark.py | 9 +- .../container/benchmarks/ml/nn_benchmark.py | 8 +- hpobench/container/benchmarks/ml/pybnn.py | 0 .../container/benchmarks/ml/rf_benchmark.py | 11 +- .../container/benchmarks/ml/svm_benchmark.py | 9 +- .../benchmarks/ml/svm_benchmark_old.py | 0 .../benchmarks/ml/tabular_benchmark.py | 0 .../benchmarks/ml/xgboost_benchmark.py | 6 + .../benchmarks/ml/xgboost_benchmark_old.py | 0 hpobench/container/benchmarks/mo/__init__.py | 0 .../benchmarks/mo/adult_benchmark.py | 0 .../container/benchmarks/mo/cnn_benchmark.py | 0 hpobench/container/benchmarks/nas/__init__.py | 0 .../benchmarks/nas/jahs_benchmarks.py | 0 .../container/benchmarks/nas/nasbench_101.py | 0 .../benchmarks/nas/nasbench_1shot1.py | 0 .../container/benchmarks/nas/nasbench_201.py | 0 .../benchmarks/nas/tabular_benchmarks.py | 0 hpobench/container/benchmarks/od/__init__.py | 0 .../container/benchmarks/od/od_benchmarks.py | 0 hpobench/container/benchmarks/rl/__init__.py | 0 hpobench/container/benchmarks/rl/cartpole.py | 0 .../benchmarks/rl/learna_benchmark.py | 0 .../benchmarks/surrogates/__init__.py | 0 .../surrogates/paramnet_benchmark.py | 0 .../benchmarks/surrogates/svm_benchmark.py | 0 .../benchmarks/surrogates/yahpo_gym.py | 0 .../container/client_abstract_benchmark.py | 10 ++ .../container/recipes/Singularity.template | 0 .../container/recipes/ml/Singularity.PyBNN | 0 .../ml/Singularity.SupportVectorMachine | 0 .../recipes/ml/Singularity.XGBoostBenchmark | 0 .../container/recipes/ml/Singularity.ml_mmfb | 0 .../ml/Singularity.ml_tabular_benchmarks | 0 .../recipes/mo/Singularity.AdultBenchmark | 0 .../recipes/mo/Singularity.CNNBenchmark | 0 .../recipes/nas/Singularity.TabularBenchmarks | 0 .../recipes/nas/Singularity.nasbench_101 | 0 .../recipes/nas/Singularity.nasbench_1shot1 | 0 .../recipes/nas/Singularity.nasbench_201 | 0 .../recipes/od/Singularity.ODBenchmarks | 0 .../od/Singularity.ODKernelDensityEstimation | 0 .../container/recipes/rl/Singularity.Cartpole | 0 .../recipes/rl/Singularity.learnaBenchmark | 0 .../surrogates/Singularity.ParamnetBenchmark | 0 .../Singularity.SupportVectorMachine | 0 .../surrogates/Singularity.YAHPOGymBenchmark | 0 .../container/server_abstract_benchmark.py | 0 hpobench/dependencies/__init__.py | 0 hpobench/dependencies/ml/__init__.py | 0 hpobench/dependencies/ml/data_manager.py | 0 .../dependencies/ml/ml_benchmark_template.py | 166 ++++++++++++++++-- hpobench/dependencies/mo/__init__.py | 0 hpobench/dependencies/mo/fairness_metrics.py | 0 hpobench/dependencies/mo/scalar.py | 0 hpobench/dependencies/od/__init__.py | 0 .../dependencies/od/backbones/__init__.py | 0 hpobench/dependencies/od/backbones/mlp.py | 0 .../dependencies/od/callbacks/__init__.py | 0 .../od/callbacks/checkpoint_saver.py | 0 .../od/callbacks/earlystopping.py | 0 hpobench/dependencies/od/data_manager.py | 0 hpobench/dependencies/od/models/__init__.py | 0 .../dependencies/od/models/autoencoder.py | 0 .../dependencies/od/traditional_benchmark.py | 0 hpobench/dependencies/od/utils/__init__.py | 0 hpobench/dependencies/od/utils/activations.py | 0 hpobench/dependencies/od/utils/scaler.py | 0 hpobench/util/__init__.py | 0 hpobench/util/clean_up_script.py | 0 hpobench/util/container_utils.py | 0 hpobench/util/data_manager.py | 0 hpobench/util/dependencies.py | 0 hpobench/util/example_utils.py | 0 hpobench/util/openml_data_manager.py | 1 + hpobench/util/rng_helper.py | 0 hpobench/util/test_utils.py | 0 requirements.txt | 0 setup.py | 0 tests/__init__.py | 0 tests/test_abstract_benchmark.py | 0 tests/test_adult.py | 9 + tests/test_check_configuration.py | 0 tests/test_container_availbable.py | 0 tests/test_data_manager.py | 0 tests/test_hpobenchconfig.py | 0 tests/test_mo_cnn.py | 0 tests/test_nasbench_101.py | 0 tests/test_nasbench_201.py | 0 tests/test_od.py | 0 tests/test_openml_datamanager.py | 0 tests/test_paramnet.py | 0 tests/test_pybnn.py | 0 tests/test_server.py | 0 tests/test_svm.py | 31 +++- tests/test_tabular_benchmarks.py | 0 tests/test_utils.py | 0 tests/test_whitebox.py | 0 tests/test_yahpo.py | 0 177 files changed, 433 insertions(+), 66 deletions(-) mode change 100644 => 100755 .github/workflows/run_singularity_versions.yml mode change 100644 => 100755 .github/workflows/run_tests.yml mode change 100644 => 100755 .gitignore mode change 100644 => 100755 LICENSE mode change 100644 => 100755 README.md mode change 100644 => 100755 changelog.md mode change 100644 => 100755 ci_scripts/codestyle.sh mode change 100644 => 100755 ci_scripts/container_examples.sh mode change 100644 => 100755 ci_scripts/install.sh mode change 100644 => 100755 ci_scripts/install_singularity.sh mode change 100644 => 100755 ci_scripts/local_examples.sh mode change 100644 => 100755 ci_scripts/script.sh mode change 100644 => 100755 codecov.yml mode change 100644 => 100755 examples/container/create_connection_to_benchmark.py mode change 100644 => 100755 examples/container/tabular_benchmark_example.py mode change 100644 => 100755 examples/container/xgboost_with_container.py mode change 100644 => 100755 examples/jahs_bench_201.json mode change 100644 => 100755 examples/local/xgboost_local.py mode change 100644 => 100755 examples/w_optimizer/SVMSurrogate_minicomparison.py mode change 100644 => 100755 examples/w_optimizer/cartpole_bohb.py mode change 100644 => 100755 examples/w_optimizer/cartpole_smachb.py mode change 100644 => 100755 extra_requirements/cartpole.json mode change 100644 => 100755 extra_requirements/examples.json mode change 100644 => 100755 extra_requirements/ml_mfbb.json mode change 100644 => 100755 extra_requirements/mo_cnn.json mode change 100644 => 100755 extra_requirements/multi_objective.json mode change 100644 => 100755 extra_requirements/nasbench_101.json mode change 100644 => 100755 extra_requirements/nasbench_1shot1.json mode change 100644 => 100755 extra_requirements/outlier_detection.json mode change 100644 => 100755 extra_requirements/paramnet.json mode change 100644 => 100755 extra_requirements/pybnn.json mode change 100644 => 100755 extra_requirements/svm.json mode change 100644 => 100755 extra_requirements/tabular_benchmarks.json mode change 100644 => 100755 extra_requirements/tests.json mode change 100644 => 100755 extra_requirements/xgboost.json mode change 100644 => 100755 extra_requirements/yahpo_gym.json mode change 100644 => 100755 hpobench/__init__.py mode change 100644 => 100755 hpobench/__version__.py mode change 100644 => 100755 hpobench/abstract_benchmark.py mode change 100644 => 100755 hpobench/benchmarks/__init__.py mode change 100644 => 100755 hpobench/benchmarks/ml/README.md mode change 100644 => 100755 hpobench/benchmarks/ml/__init__.py mode change 100644 => 100755 hpobench/benchmarks/ml/histgb_benchmark.py mode change 100644 => 100755 hpobench/benchmarks/ml/lr_benchmark.py mode change 100644 => 100755 hpobench/benchmarks/ml/nn_benchmark.py mode change 100644 => 100755 hpobench/benchmarks/ml/pybnn.py mode change 100644 => 100755 hpobench/benchmarks/ml/rf_benchmark.py mode change 100644 => 100755 hpobench/benchmarks/ml/svm_benchmark.py mode change 100644 => 100755 hpobench/benchmarks/ml/svm_benchmark_old.py mode change 100644 => 100755 hpobench/benchmarks/ml/tabular_benchmark.py mode change 100644 => 100755 hpobench/benchmarks/ml/xgboost_benchmark.py mode change 100644 => 100755 hpobench/benchmarks/ml/xgboost_benchmark_old.py mode change 100644 => 100755 hpobench/benchmarks/mo/__init__.py mode change 100644 => 100755 hpobench/benchmarks/mo/adult_benchmark.py mode change 100644 => 100755 hpobench/benchmarks/mo/cnn_benchmark.py mode change 100644 => 100755 hpobench/benchmarks/nas/__init__.py mode change 100644 => 100755 hpobench/benchmarks/nas/jahs_benchmarks.py mode change 100644 => 100755 hpobench/benchmarks/nas/nasbench_101.py mode change 100644 => 100755 hpobench/benchmarks/nas/nasbench_1shot1.py mode change 100644 => 100755 hpobench/benchmarks/nas/nasbench_201.py mode change 100644 => 100755 hpobench/benchmarks/nas/tabular_benchmarks.py mode change 100644 => 100755 hpobench/benchmarks/od/__init__.py mode change 100644 => 100755 hpobench/benchmarks/od/od_ae.py mode change 100644 => 100755 hpobench/benchmarks/od/od_benchmarks.py mode change 100644 => 100755 hpobench/benchmarks/od/od_kde.py mode change 100644 => 100755 hpobench/benchmarks/od/od_ocsvm.py mode change 100644 => 100755 hpobench/benchmarks/rl/__init__.py mode change 100644 => 100755 hpobench/benchmarks/rl/cartpole.py mode change 100644 => 100755 hpobench/benchmarks/rl/learna_benchmark.py mode change 100644 => 100755 hpobench/benchmarks/surrogates/__init__.py mode change 100644 => 100755 hpobench/benchmarks/surrogates/paramnet_benchmark.py mode change 100644 => 100755 hpobench/benchmarks/surrogates/svm_benchmark.py mode change 100644 => 100755 hpobench/benchmarks/surrogates/yahpo_gym.py mode change 100644 => 100755 hpobench/config.py mode change 100644 => 100755 hpobench/container/__init__.py mode change 100644 => 100755 hpobench/container/benchmarks/__init__.py mode change 100644 => 100755 hpobench/container/benchmarks/ml/__init__.py mode change 100644 => 100755 hpobench/container/benchmarks/ml/histgb_benchmark.py mode change 100644 => 100755 hpobench/container/benchmarks/ml/lr_benchmark.py mode change 100644 => 100755 hpobench/container/benchmarks/ml/nn_benchmark.py mode change 100644 => 100755 hpobench/container/benchmarks/ml/pybnn.py mode change 100644 => 100755 hpobench/container/benchmarks/ml/rf_benchmark.py mode change 100644 => 100755 hpobench/container/benchmarks/ml/svm_benchmark.py mode change 100644 => 100755 hpobench/container/benchmarks/ml/svm_benchmark_old.py mode change 100644 => 100755 hpobench/container/benchmarks/ml/tabular_benchmark.py mode change 100644 => 100755 hpobench/container/benchmarks/ml/xgboost_benchmark.py mode change 100644 => 100755 hpobench/container/benchmarks/ml/xgboost_benchmark_old.py mode change 100644 => 100755 hpobench/container/benchmarks/mo/__init__.py mode change 100644 => 100755 hpobench/container/benchmarks/mo/adult_benchmark.py mode change 100644 => 100755 hpobench/container/benchmarks/mo/cnn_benchmark.py mode change 100644 => 100755 hpobench/container/benchmarks/nas/__init__.py mode change 100644 => 100755 hpobench/container/benchmarks/nas/jahs_benchmarks.py mode change 100644 => 100755 hpobench/container/benchmarks/nas/nasbench_101.py mode change 100644 => 100755 hpobench/container/benchmarks/nas/nasbench_1shot1.py mode change 100644 => 100755 hpobench/container/benchmarks/nas/nasbench_201.py mode change 100644 => 100755 hpobench/container/benchmarks/nas/tabular_benchmarks.py mode change 100644 => 100755 hpobench/container/benchmarks/od/__init__.py mode change 100644 => 100755 hpobench/container/benchmarks/od/od_benchmarks.py mode change 100644 => 100755 hpobench/container/benchmarks/rl/__init__.py mode change 100644 => 100755 hpobench/container/benchmarks/rl/cartpole.py mode change 100644 => 100755 hpobench/container/benchmarks/rl/learna_benchmark.py mode change 100644 => 100755 hpobench/container/benchmarks/surrogates/__init__.py mode change 100644 => 100755 hpobench/container/benchmarks/surrogates/paramnet_benchmark.py mode change 100644 => 100755 hpobench/container/benchmarks/surrogates/svm_benchmark.py mode change 100644 => 100755 hpobench/container/benchmarks/surrogates/yahpo_gym.py mode change 100644 => 100755 hpobench/container/client_abstract_benchmark.py mode change 100644 => 100755 hpobench/container/recipes/Singularity.template mode change 100644 => 100755 hpobench/container/recipes/ml/Singularity.PyBNN mode change 100644 => 100755 hpobench/container/recipes/ml/Singularity.SupportVectorMachine mode change 100644 => 100755 hpobench/container/recipes/ml/Singularity.XGBoostBenchmark mode change 100644 => 100755 hpobench/container/recipes/ml/Singularity.ml_mmfb mode change 100644 => 100755 hpobench/container/recipes/ml/Singularity.ml_tabular_benchmarks mode change 100644 => 100755 hpobench/container/recipes/mo/Singularity.AdultBenchmark mode change 100644 => 100755 hpobench/container/recipes/mo/Singularity.CNNBenchmark mode change 100644 => 100755 hpobench/container/recipes/nas/Singularity.TabularBenchmarks mode change 100644 => 100755 hpobench/container/recipes/nas/Singularity.nasbench_101 mode change 100644 => 100755 hpobench/container/recipes/nas/Singularity.nasbench_1shot1 mode change 100644 => 100755 hpobench/container/recipes/nas/Singularity.nasbench_201 mode change 100644 => 100755 hpobench/container/recipes/od/Singularity.ODBenchmarks mode change 100644 => 100755 hpobench/container/recipes/od/Singularity.ODKernelDensityEstimation mode change 100644 => 100755 hpobench/container/recipes/rl/Singularity.Cartpole mode change 100644 => 100755 hpobench/container/recipes/rl/Singularity.learnaBenchmark mode change 100644 => 100755 hpobench/container/recipes/surrogates/Singularity.ParamnetBenchmark mode change 100644 => 100755 hpobench/container/recipes/surrogates/Singularity.SupportVectorMachine mode change 100644 => 100755 hpobench/container/recipes/surrogates/Singularity.YAHPOGymBenchmark mode change 100644 => 100755 hpobench/container/server_abstract_benchmark.py mode change 100644 => 100755 hpobench/dependencies/__init__.py mode change 100644 => 100755 hpobench/dependencies/ml/__init__.py mode change 100644 => 100755 hpobench/dependencies/ml/data_manager.py mode change 100644 => 100755 hpobench/dependencies/ml/ml_benchmark_template.py mode change 100644 => 100755 hpobench/dependencies/mo/__init__.py mode change 100644 => 100755 hpobench/dependencies/mo/fairness_metrics.py mode change 100644 => 100755 hpobench/dependencies/mo/scalar.py mode change 100644 => 100755 hpobench/dependencies/od/__init__.py mode change 100644 => 100755 hpobench/dependencies/od/backbones/__init__.py mode change 100644 => 100755 hpobench/dependencies/od/backbones/mlp.py mode change 100644 => 100755 hpobench/dependencies/od/callbacks/__init__.py mode change 100644 => 100755 hpobench/dependencies/od/callbacks/checkpoint_saver.py mode change 100644 => 100755 hpobench/dependencies/od/callbacks/earlystopping.py mode change 100644 => 100755 hpobench/dependencies/od/data_manager.py mode change 100644 => 100755 hpobench/dependencies/od/models/__init__.py mode change 100644 => 100755 hpobench/dependencies/od/models/autoencoder.py mode change 100644 => 100755 hpobench/dependencies/od/traditional_benchmark.py mode change 100644 => 100755 hpobench/dependencies/od/utils/__init__.py mode change 100644 => 100755 hpobench/dependencies/od/utils/activations.py mode change 100644 => 100755 hpobench/dependencies/od/utils/scaler.py mode change 100644 => 100755 hpobench/util/__init__.py mode change 100644 => 100755 hpobench/util/clean_up_script.py mode change 100644 => 100755 hpobench/util/container_utils.py mode change 100644 => 100755 hpobench/util/data_manager.py mode change 100644 => 100755 hpobench/util/dependencies.py mode change 100644 => 100755 hpobench/util/example_utils.py mode change 100644 => 100755 hpobench/util/openml_data_manager.py mode change 100644 => 100755 hpobench/util/rng_helper.py mode change 100644 => 100755 hpobench/util/test_utils.py mode change 100644 => 100755 requirements.txt mode change 100644 => 100755 setup.py mode change 100644 => 100755 tests/__init__.py mode change 100644 => 100755 tests/test_abstract_benchmark.py mode change 100644 => 100755 tests/test_adult.py mode change 100644 => 100755 tests/test_check_configuration.py mode change 100644 => 100755 tests/test_container_availbable.py mode change 100644 => 100755 tests/test_data_manager.py mode change 100644 => 100755 tests/test_hpobenchconfig.py mode change 100644 => 100755 tests/test_mo_cnn.py mode change 100644 => 100755 tests/test_nasbench_101.py mode change 100644 => 100755 tests/test_nasbench_201.py mode change 100644 => 100755 tests/test_od.py mode change 100644 => 100755 tests/test_openml_datamanager.py mode change 100644 => 100755 tests/test_paramnet.py mode change 100644 => 100755 tests/test_pybnn.py mode change 100644 => 100755 tests/test_server.py mode change 100644 => 100755 tests/test_svm.py mode change 100644 => 100755 tests/test_tabular_benchmarks.py mode change 100644 => 100755 tests/test_utils.py mode change 100644 => 100755 tests/test_whitebox.py mode change 100644 => 100755 tests/test_yahpo.py diff --git a/.github/workflows/run_singularity_versions.yml b/.github/workflows/run_singularity_versions.yml old mode 100644 new mode 100755 diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml old mode 100644 new mode 100755 diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/changelog.md b/changelog.md old mode 100644 new mode 100755 diff --git a/ci_scripts/codestyle.sh b/ci_scripts/codestyle.sh old mode 100644 new mode 100755 diff --git a/ci_scripts/container_examples.sh b/ci_scripts/container_examples.sh old mode 100644 new mode 100755 diff --git a/ci_scripts/install.sh b/ci_scripts/install.sh old mode 100644 new mode 100755 diff --git a/ci_scripts/install_singularity.sh b/ci_scripts/install_singularity.sh old mode 100644 new mode 100755 diff --git a/ci_scripts/local_examples.sh b/ci_scripts/local_examples.sh old mode 100644 new mode 100755 diff --git a/ci_scripts/script.sh b/ci_scripts/script.sh old mode 100644 new mode 100755 diff --git a/codecov.yml b/codecov.yml old mode 100644 new mode 100755 diff --git a/examples/container/create_connection_to_benchmark.py b/examples/container/create_connection_to_benchmark.py old mode 100644 new mode 100755 diff --git a/examples/container/tabular_benchmark_example.py b/examples/container/tabular_benchmark_example.py old mode 100644 new mode 100755 diff --git a/examples/container/xgboost_with_container.py b/examples/container/xgboost_with_container.py old mode 100644 new mode 100755 diff --git a/examples/jahs_bench_201.json b/examples/jahs_bench_201.json old mode 100644 new mode 100755 diff --git a/examples/local/xgboost_local.py b/examples/local/xgboost_local.py old mode 100644 new mode 100755 diff --git a/examples/w_optimizer/SVMSurrogate_minicomparison.py b/examples/w_optimizer/SVMSurrogate_minicomparison.py old mode 100644 new mode 100755 diff --git a/examples/w_optimizer/cartpole_bohb.py b/examples/w_optimizer/cartpole_bohb.py old mode 100644 new mode 100755 diff --git a/examples/w_optimizer/cartpole_smachb.py b/examples/w_optimizer/cartpole_smachb.py old mode 100644 new mode 100755 diff --git a/extra_requirements/cartpole.json b/extra_requirements/cartpole.json old mode 100644 new mode 100755 diff --git a/extra_requirements/examples.json b/extra_requirements/examples.json old mode 100644 new mode 100755 diff --git a/extra_requirements/ml_mfbb.json b/extra_requirements/ml_mfbb.json old mode 100644 new mode 100755 diff --git a/extra_requirements/mo_cnn.json b/extra_requirements/mo_cnn.json old mode 100644 new mode 100755 diff --git a/extra_requirements/multi_objective.json b/extra_requirements/multi_objective.json old mode 100644 new mode 100755 diff --git a/extra_requirements/nasbench_101.json b/extra_requirements/nasbench_101.json old mode 100644 new mode 100755 diff --git a/extra_requirements/nasbench_1shot1.json b/extra_requirements/nasbench_1shot1.json old mode 100644 new mode 100755 diff --git a/extra_requirements/outlier_detection.json b/extra_requirements/outlier_detection.json old mode 100644 new mode 100755 diff --git a/extra_requirements/paramnet.json b/extra_requirements/paramnet.json old mode 100644 new mode 100755 diff --git a/extra_requirements/pybnn.json b/extra_requirements/pybnn.json old mode 100644 new mode 100755 diff --git a/extra_requirements/svm.json b/extra_requirements/svm.json old mode 100644 new mode 100755 diff --git a/extra_requirements/tabular_benchmarks.json b/extra_requirements/tabular_benchmarks.json old mode 100644 new mode 100755 diff --git a/extra_requirements/tests.json b/extra_requirements/tests.json old mode 100644 new mode 100755 diff --git a/extra_requirements/xgboost.json b/extra_requirements/xgboost.json old mode 100644 new mode 100755 diff --git a/extra_requirements/yahpo_gym.json b/extra_requirements/yahpo_gym.json old mode 100644 new mode 100755 diff --git a/hpobench/__init__.py b/hpobench/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/__version__.py b/hpobench/__version__.py old mode 100644 new mode 100755 diff --git a/hpobench/abstract_benchmark.py b/hpobench/abstract_benchmark.py old mode 100644 new mode 100755 index 6a2942af..bcc57542 --- a/hpobench/abstract_benchmark.py +++ b/hpobench/abstract_benchmark.py @@ -35,6 +35,7 @@ def __init__(self, rng: Union[int, np.random.RandomState, None] = None, **kwargs create a new random state. """ super(_BaseAbstractBenchmark, self).__init__(**kwargs) + print("base abstract benchamrk") self.rng = rng_helper.get_rng(rng=rng) self.configuration_space = self.get_configuration_space(self.rng.randint(0, 10000)) self.fidelity_space = self.get_fidelity_space(self.rng.randint(0, 10000)) diff --git a/hpobench/benchmarks/__init__.py b/hpobench/benchmarks/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/ml/README.md b/hpobench/benchmarks/ml/README.md old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/ml/__init__.py b/hpobench/benchmarks/ml/__init__.py old mode 100644 new mode 100755 index 64e399cd..4c0a1db0 --- a/hpobench/benchmarks/ml/__init__.py +++ b/hpobench/benchmarks/ml/__init__.py @@ -1,22 +1,22 @@ -from hpobench.benchmarks.ml.histgb_benchmark import HistGBBenchmark, HistGBBenchmarkBB, HistGBBenchmarkMF -from hpobench.benchmarks.ml.lr_benchmark import LRBenchmark, LRBenchmarkBB, LRBenchmarkMF -from hpobench.benchmarks.ml.nn_benchmark import NNBenchmark, NNBenchmarkBB, NNBenchmarkMF +from hpobench.benchmarks.ml.histgb_benchmark import HistGBBenchmark, HistGBBenchmarkBB, HistGBBenchmarkMF, HistGBBenchmarkMO +from hpobench.benchmarks.ml.lr_benchmark import LRBenchmark, LRBenchmarkBB, LRBenchmarkMF, LRBenchmarkMO +from hpobench.benchmarks.ml.nn_benchmark import NNBenchmark, NNBenchmarkBB, NNBenchmarkMF, NNBenchmarkMO from hpobench.benchmarks.ml.rf_benchmark import RandomForestBenchmark, RandomForestBenchmarkBB, \ - RandomForestBenchmarkMF -from hpobench.benchmarks.ml.svm_benchmark import SVMBenchmark, SVMBenchmarkBB, SVMBenchmarkMF + RandomForestBenchmarkMF, RandomForestBenchmarkMO +from hpobench.benchmarks.ml.svm_benchmark import SVMBenchmark, SVMBenchmarkBB, SVMBenchmarkMF, SVMBenchmarkMO from hpobench.benchmarks.ml.tabular_benchmark import TabularBenchmark try: - from hpobench.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark, XGBoostBenchmarkBB, XGBoostBenchmarkMF + from hpobench.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark, XGBoostBenchmarkBB, XGBoostBenchmarkMF, XGBoostBenchmarkMO except ImportError: pass -__all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF', - 'LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF', - 'NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF', - 'RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF', - 'SVMBenchmark', 'SVMBenchmarkBB', 'SVMBenchmarkMF', +__all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF', 'HistGBBenchmarkMO', + 'LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF', 'LRBenchmarkMO', + 'NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF', 'NNBenchmarkMO', + 'RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF', 'RandomForestBenchmarkMO', + 'SVMBenchmark', 'SVMBenchmarkBB', 'SVMBenchmarkMF', 'SVMBenchmarkMO', 'TabularBenchmark', - 'XGBoostBenchmark', 'XGBoostBenchmarkBB', 'XGBoostBenchmarkMF', + 'XGBoostBenchmark', 'XGBoostBenchmarkBB', 'XGBoostBenchmarkMF', 'XGBoostBenchmarkMO', ] diff --git a/hpobench/benchmarks/ml/histgb_benchmark.py b/hpobench/benchmarks/ml/histgb_benchmark.py old mode 100644 new mode 100755 index c4b0c468..097fefaa --- a/hpobench/benchmarks/ml/histgb_benchmark.py +++ b/hpobench/benchmarks/ml/histgb_benchmark.py @@ -15,18 +15,18 @@ from sklearn.experimental import enable_hist_gradient_boosting # noqa from sklearn.ensemble import HistGradientBoostingClassifier -from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark +from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark, MO_MLBenchmark __version__ = '0.0.1' -class HistGBBenchmark(MLBenchmark): +class _HistGBBenchmarkBase: def __init__(self, task_id: int, rng: Union[np.random.RandomState, int, None] = None, valid_size: float = 0.33, data_path: Union[str, None] = None): - super(HistGBBenchmark, self).__init__(task_id, rng, valid_size, data_path) + super(_HistGBBenchmarkBase, self).__init__(task_id, rng, valid_size, data_path) @staticmethod def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -101,6 +101,13 @@ def init_model(self, config: Union[CS.Configuration, Dict], ) return model +class HistGBBenchmark(_HistGBBenchmarkBase, MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(HistGBBenchmark, self).__init__(task_id, rng, valid_size, data_path) class HistGBBenchmarkBB(HistGBBenchmark): def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -120,6 +127,14 @@ def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationS HistGBBenchmark._get_fidelity_choices(ntrees_choice='variable', subsample_choice='fixed') ) return fidelity_space + +class HistGBBenchmarkMO(_HistGBBenchmarkBase, MO_MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(HistGBBenchmarkMO, self).__init__(task_id, rng, valid_size, data_path) -__all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF'] +__all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF', 'HistGBBenchmarkMO'] diff --git a/hpobench/benchmarks/ml/lr_benchmark.py b/hpobench/benchmarks/ml/lr_benchmark.py old mode 100644 new mode 100755 index 8c317111..b50e8f35 --- a/hpobench/benchmarks/ml/lr_benchmark.py +++ b/hpobench/benchmarks/ml/lr_benchmark.py @@ -14,19 +14,28 @@ from ConfigSpace.hyperparameters import Hyperparameter from sklearn.linear_model import SGDClassifier -from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark +from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark, MO_MLBenchmark +''' +Changelog: +========== +0.0.2: +* Add the multiobjective version of this benchmark by returning val loss, precision, f1 and balanced accuracy + +0.0.1: +* First implementation +''' -__version__ = '0.0.1' +__version__ = '0.0.2' -class LRBenchmark(MLBenchmark): +class _LRBenchmarkBase: def __init__(self, task_id: int, rng: Union[np.random.RandomState, int, None] = None, valid_size: float = 0.33, data_path: Union[str, None] = None): - super(LRBenchmark, self).__init__(task_id, rng, valid_size, data_path) + super(_LRBenchmarkBase, self).__init__(task_id, rng, valid_size, data_path) self.cache_size = 500 @staticmethod @@ -107,6 +116,17 @@ def init_model(self, config: Union[CS.Configuration, Dict], ) return model +class LRBenchmark(_LRBenchmarkBase, MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + + super(LRBenchmark, self).__init__(task_id, rng, valid_size, data_path) + self.cache_size = 500 + + class LRBenchmarkBB(LRBenchmark): def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -126,6 +146,16 @@ def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationS LRBenchmark._get_fidelity_choices(iter_choice='variable', subsample_choice='fixed') ) return fidelity_space + +class LRBenchmarkMO(_LRBenchmarkBase, MO_MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + + super(LRBenchmark, self).__init__(task_id, rng, valid_size, data_path) + self.cache_size = 500 -__all__ = ['LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF'] +__all__ = ['LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF', 'LRBenchmarkMO'] diff --git a/hpobench/benchmarks/ml/nn_benchmark.py b/hpobench/benchmarks/ml/nn_benchmark.py old mode 100644 new mode 100755 index 06634661..d22d2498 --- a/hpobench/benchmarks/ml/nn_benchmark.py +++ b/hpobench/benchmarks/ml/nn_benchmark.py @@ -14,18 +14,28 @@ from ConfigSpace.hyperparameters import Hyperparameter from sklearn.neural_network import MLPClassifier -from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark +from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark, MO_MLBenchmark -__version__ = '0.0.1' +''' +Changelog: +========== +0.0.2: +* Add the multiobjective version of this benchmark by returning val loss, precision, f1 and balanced accuracy + +0.0.1: +* First implementation +''' + +__version__ = '0.0.2' -class NNBenchmark(MLBenchmark): +class _NNBenchmarkBase: def __init__(self, task_id: int, rng: Union[np.random.RandomState, int, None] = None, valid_size: float = 0.33, data_path: Union[str, None] = None): - super(NNBenchmark, self).__init__(task_id, rng, valid_size, data_path) + super(_NNBenchmarkBase, self).__init__(task_id, rng, valid_size, data_path) @staticmethod def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -109,6 +119,13 @@ def init_model(self, config: Union[CS.Configuration, Dict], ) return model +class NNBenchmark(_NNBenchmarkBase, MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(NNBenchmark, self).__init__(task_id, rng, valid_size, data_path) class NNBenchmarkBB(NNBenchmark): def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -129,5 +146,12 @@ def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationS ) return fidelity_space +class NNBenchmarkMO(_NNBenchmarkBase, MO_MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(NNBenchmark, self).__init__(task_id, rng, valid_size, data_path) -__all__ = ['NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF'] +__all__ = ['NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF', 'NNBenchmarkMO'] diff --git a/hpobench/benchmarks/ml/pybnn.py b/hpobench/benchmarks/ml/pybnn.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/ml/rf_benchmark.py b/hpobench/benchmarks/ml/rf_benchmark.py old mode 100644 new mode 100755 index 596f03b6..7acce3c6 --- a/hpobench/benchmarks/ml/rf_benchmark.py +++ b/hpobench/benchmarks/ml/rf_benchmark.py @@ -14,18 +14,27 @@ from ConfigSpace.hyperparameters import Hyperparameter from sklearn.ensemble import RandomForestClassifier -from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark +from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark, MO_MLBenchmark -__version__ = '0.0.1' +''' +Changelog: +========== +0.0.2: +* Add the multiobjective version of this benchmark by returning val loss, precision, f1 and balanced accuracy +0.0.1: +* First implementation +''' +__version__ = '0.0.2' -class RandomForestBenchmark(MLBenchmark): + +class _RandomForestBenchmarkBase: def __init__(self, task_id: int, rng: Union[np.random.RandomState, int, None] = None, valid_size: float = 0.33, data_path: Union[str, None] = None): - super(RandomForestBenchmark, self).__init__(task_id, rng, valid_size, data_path) + super(_RandomForestBenchmarkBase, self).__init__(task_id, rng, valid_size, data_path) @staticmethod def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -103,6 +112,13 @@ def init_model(self, config: Union[CS.Configuration, Dict], ) return model +class RandomForestBenchmark(_RandomForestBenchmarkBase, MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(RandomForestBenchmark, self).__init__(task_id, rng, valid_size, data_path) class RandomForestBenchmarkBB(RandomForestBenchmark): def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -122,6 +138,17 @@ def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationS RandomForestBenchmark._get_fidelity_choices(n_estimators_choice='variable', subsample_choice='fixed') ) return fidelity_space + +class RandomForestBenchmarkMO(_RandomForestBenchmarkBase, MO_MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(RandomForestBenchmarkMO, self).__init__(task_id, rng, valid_size, data_path) + + -__all__ = ['RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF'] +__all__ = ['RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF', + 'RandomForestBenchmarkMO'] diff --git a/hpobench/benchmarks/ml/svm_benchmark.py b/hpobench/benchmarks/ml/svm_benchmark.py old mode 100644 new mode 100755 index 9462442f..e77f1b64 --- a/hpobench/benchmarks/ml/svm_benchmark.py +++ b/hpobench/benchmarks/ml/svm_benchmark.py @@ -13,18 +13,27 @@ from ConfigSpace.hyperparameters import Hyperparameter from sklearn.svm import SVC -from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark +from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark, MO_MLBenchmark +''' +Changelog: +========== +0.0.2: +* Add the multiobjective version of this benchmark by returning val loss, precision, f1 and balanced accuracy -__version__ = '0.0.1' +0.0.1: +* First implementation +''' +__version__ = '0.0.2' -class SVMBenchmark(MLBenchmark): + +class _SVMBenchmarkBase: def __init__(self, task_id: int, rng: Union[np.random.RandomState, int, None] = None, valid_size: float = 0.33, data_path: Union[str, None] = None): - super(SVMBenchmark, self).__init__(task_id, rng, valid_size, data_path) + super(_SVMBenchmarkBase, self).__init__(task_id, rng, valid_size, data_path) self.cache_size = 200 @@ -81,6 +90,14 @@ def init_model(self, config: Union[CS.Configuration, Dict], ) return model +class SVMBenchmark(_SVMBenchmarkBase, MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(SVMBenchmark, self).__init__(task_id, rng, valid_size, data_path) + class SVMBenchmarkBB(SVMBenchmark): def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -90,9 +107,19 @@ def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationS SVMBenchmark._get_fidelity_choices(subsample_choice='fixed') ) return fidelity_space + +class SVMBenchmarkMO(_SVMBenchmarkBase, MO_MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(SVMBenchmarkMO, self).__init__(task_id, rng, valid_size, data_path) + + # To keep the parity of the the overall design SVMBenchmarkMF = SVMBenchmark -__all__ = ['SVMBenchmark', 'SVMBenchmarkMF', 'SVMBenchmarkBB'] +__all__ = ['SVMBenchmark', 'SVMBenchmarkMF', 'SVMBenchmarkBB', 'SVMBenchmarkMO'] diff --git a/hpobench/benchmarks/ml/svm_benchmark_old.py b/hpobench/benchmarks/ml/svm_benchmark_old.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/ml/tabular_benchmark.py b/hpobench/benchmarks/ml/tabular_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/ml/xgboost_benchmark.py b/hpobench/benchmarks/ml/xgboost_benchmark.py old mode 100644 new mode 100755 index ae554628..f2d65a99 --- a/hpobench/benchmarks/ml/xgboost_benchmark.py +++ b/hpobench/benchmarks/ml/xgboost_benchmark.py @@ -12,18 +12,27 @@ import xgboost as xgb from ConfigSpace.hyperparameters import Hyperparameter -from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark +from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark, MO_MLBenchmark -__version__ = '0.0.1' +''' +Changelog: +========== +0.0.2: +* Add the multiobjective version of this benchmark by returning val loss, precision, f1 and balanced accuracy +0.0.1: +* First implementation +''' +__version__ = '0.0.2' -class XGBoostBenchmark(MLBenchmark): + +class _XGBoostBenchmarkBase: def __init__(self, task_id: int, rng: Union[np.random.RandomState, int, None] = None, valid_size: float = 0.33, data_path: Union[str, None] = None): - super(XGBoostBenchmark, self).__init__(task_id, rng, valid_size, data_path) + super(_XGBoostBenchmarkBase, self).__init__(task_id, rng, valid_size, data_path) @staticmethod def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -108,6 +117,13 @@ def init_model(self, ) return model +class XGBoostBenchmark(_XGBoostBenchmarkBase, MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(XGBoostBenchmark, self).__init__(task_id, rng, valid_size, data_path) class XGBoostBenchmarkBB(XGBoostBenchmark): def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -127,6 +143,15 @@ def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationS XGBoostBenchmark._get_fidelity_choices(n_estimators_choice='variable', subsample_choice='fixed') ) return fidelity_space + +class XGBoostBenchmarkMO(_XGBoostBenchmarkBase, MO_MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(XGBoostBenchmarkMO, self).__init__(task_id, rng, valid_size, data_path) + -__all__ = ['XGBoostBenchmarkBB', 'XGBoostBenchmarkMF', 'XGBoostBenchmark'] +__all__ = ['XGBoostBenchmarkBB', 'XGBoostBenchmarkMF', 'XGBoostBenchmark', 'XGBoostBenchmarkMO'] diff --git a/hpobench/benchmarks/ml/xgboost_benchmark_old.py b/hpobench/benchmarks/ml/xgboost_benchmark_old.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/mo/__init__.py b/hpobench/benchmarks/mo/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/mo/adult_benchmark.py b/hpobench/benchmarks/mo/adult_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/mo/cnn_benchmark.py b/hpobench/benchmarks/mo/cnn_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/nas/__init__.py b/hpobench/benchmarks/nas/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/nas/jahs_benchmarks.py b/hpobench/benchmarks/nas/jahs_benchmarks.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/nas/nasbench_101.py b/hpobench/benchmarks/nas/nasbench_101.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/nas/nasbench_1shot1.py b/hpobench/benchmarks/nas/nasbench_1shot1.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/nas/nasbench_201.py b/hpobench/benchmarks/nas/nasbench_201.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/nas/tabular_benchmarks.py b/hpobench/benchmarks/nas/tabular_benchmarks.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/od/__init__.py b/hpobench/benchmarks/od/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/od/od_ae.py b/hpobench/benchmarks/od/od_ae.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/od/od_benchmarks.py b/hpobench/benchmarks/od/od_benchmarks.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/od/od_kde.py b/hpobench/benchmarks/od/od_kde.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/od/od_ocsvm.py b/hpobench/benchmarks/od/od_ocsvm.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/rl/__init__.py b/hpobench/benchmarks/rl/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/rl/cartpole.py b/hpobench/benchmarks/rl/cartpole.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/rl/learna_benchmark.py b/hpobench/benchmarks/rl/learna_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/surrogates/__init__.py b/hpobench/benchmarks/surrogates/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/surrogates/paramnet_benchmark.py b/hpobench/benchmarks/surrogates/paramnet_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/surrogates/svm_benchmark.py b/hpobench/benchmarks/surrogates/svm_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/surrogates/yahpo_gym.py b/hpobench/benchmarks/surrogates/yahpo_gym.py old mode 100644 new mode 100755 diff --git a/hpobench/config.py b/hpobench/config.py old mode 100644 new mode 100755 diff --git a/hpobench/container/__init__.py b/hpobench/container/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/__init__.py b/hpobench/container/benchmarks/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/ml/__init__.py b/hpobench/container/benchmarks/ml/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/ml/histgb_benchmark.py b/hpobench/container/benchmarks/ml/histgb_benchmark.py old mode 100644 new mode 100755 index dc7af088..604c9fe4 --- a/hpobench/container/benchmarks/ml/histgb_benchmark.py +++ b/hpobench/container/benchmarks/ml/histgb_benchmark.py @@ -13,6 +13,12 @@ def __init__(self, **kwargs): kwargs['latest'] = kwargs.get('container_tag', '0.0.1') super(HistGBBenchmark, self).__init__(**kwargs) +class HistGBBenchmarkMO(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'HistGBBenchmarkMO') + kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') + kwargs['latest'] = kwargs.get('container_tag', '0.0.2') + super(HistGBBenchmark, self).__init__(**kwargs) class HistGBBenchmarkBB(AbstractBenchmarkClient): def __init__(self, **kwargs): @@ -30,4 +36,4 @@ def __init__(self, **kwargs): super(HistGBBenchmarkMF, self).__init__(**kwargs) -__all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF'] +__all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF', 'HistGBBenchmarkMO'] diff --git a/hpobench/container/benchmarks/ml/lr_benchmark.py b/hpobench/container/benchmarks/ml/lr_benchmark.py old mode 100644 new mode 100755 index 979cda3e..0d76e80b --- a/hpobench/container/benchmarks/ml/lr_benchmark.py +++ b/hpobench/container/benchmarks/ml/lr_benchmark.py @@ -13,6 +13,13 @@ def __init__(self, **kwargs): kwargs['latest'] = kwargs.get('container_tag', '0.0.1') super(LRBenchmark, self).__init__(**kwargs) +class LRBenchmarkMO(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'LRBenchmarkMO') + kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') + kwargs['latest'] = kwargs.get('container_tag', '0.0.2') + super(LRBenchmarkMO, self).__init__(**kwargs) + class LRBenchmarkBB(AbstractBenchmarkClient): def __init__(self, **kwargs): @@ -30,4 +37,4 @@ def __init__(self, **kwargs): super(LRBenchmarkMF, self).__init__(**kwargs) -__all__ = ['LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF'] +__all__ = ['LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF', 'LRBenchmarkMO'] diff --git a/hpobench/container/benchmarks/ml/nn_benchmark.py b/hpobench/container/benchmarks/ml/nn_benchmark.py old mode 100644 new mode 100755 index 04955e82..65138f80 --- a/hpobench/container/benchmarks/ml/nn_benchmark.py +++ b/hpobench/container/benchmarks/ml/nn_benchmark.py @@ -13,6 +13,12 @@ def __init__(self, **kwargs): kwargs['latest'] = kwargs.get('container_tag', '0.0.1') super(NNBenchmark, self).__init__(**kwargs) +class NNBenchmarkMO(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'NNBenchmarkMO') + kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') + kwargs['latest'] = kwargs.get('container_tag', '0.0.2') + super(NNBenchmarkMO, self).__init__(**kwargs) class NNBenchmarkBB(AbstractBenchmarkClient): def __init__(self, **kwargs): @@ -30,4 +36,4 @@ def __init__(self, **kwargs): super(NNBenchmarkMF, self).__init__(**kwargs) -__all__ = ['NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF'] +__all__ = ['NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF', 'NNBenchmarkMO'] diff --git a/hpobench/container/benchmarks/ml/pybnn.py b/hpobench/container/benchmarks/ml/pybnn.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/ml/rf_benchmark.py b/hpobench/container/benchmarks/ml/rf_benchmark.py old mode 100644 new mode 100755 index a414349d..35c208e5 --- a/hpobench/container/benchmarks/ml/rf_benchmark.py +++ b/hpobench/container/benchmarks/ml/rf_benchmark.py @@ -14,6 +14,15 @@ def __init__(self, **kwargs): super(RandomForestBenchmark, self).__init__(**kwargs) + +class RandomForestBenchmarkMO(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'RandomForestBenchmarkMO') + kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') + kwargs['latest'] = kwargs.get('container_tag', '0.0.2') + super(RandomForestBenchmarkMO, self).__init__(**kwargs) + + class RandomForestBenchmarkBB(AbstractBenchmarkClient): def __init__(self, **kwargs): kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'RandomForestBenchmarkBB') @@ -30,4 +39,4 @@ def __init__(self, **kwargs): super(RandomForestBenchmarkMF, self).__init__(**kwargs) -__all__ = ['RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF'] +__all__ = ['RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF', 'RandomForestBenchmarkMO'] diff --git a/hpobench/container/benchmarks/ml/svm_benchmark.py b/hpobench/container/benchmarks/ml/svm_benchmark.py old mode 100644 new mode 100755 index 7547a81a..ae6ff865 --- a/hpobench/container/benchmarks/ml/svm_benchmark.py +++ b/hpobench/container/benchmarks/ml/svm_benchmark.py @@ -13,6 +13,13 @@ def __init__(self, **kwargs): kwargs['latest'] = kwargs.get('container_tag', '0.0.1') super(SVMBenchmark, self).__init__(**kwargs) +class SVMBenchmarkMO(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'SVMBenchmarkMO') + kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') + kwargs['latest'] = kwargs.get('container_tag', '0.0.2') + super(SVMBenchmarkMO, self).__init__(**kwargs) + class SVMBenchmarkMF(AbstractBenchmarkClient): def __init__(self, **kwargs): @@ -30,4 +37,4 @@ def __init__(self, **kwargs): super(SVMBenchmarkBB, self).__init__(**kwargs) -__all__ = ['SVMBenchmark', 'SVMBenchmarkMF', 'SVMBenchmarkBB'] +__all__ = ['SVMBenchmark', 'SVMBenchmarkMF', 'SVMBenchmarkBB', 'SVMBenchmarkMO'] diff --git a/hpobench/container/benchmarks/ml/svm_benchmark_old.py b/hpobench/container/benchmarks/ml/svm_benchmark_old.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/ml/tabular_benchmark.py b/hpobench/container/benchmarks/ml/tabular_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/ml/xgboost_benchmark.py b/hpobench/container/benchmarks/ml/xgboost_benchmark.py old mode 100644 new mode 100755 index c82ea606..38c85c8e --- a/hpobench/container/benchmarks/ml/xgboost_benchmark.py +++ b/hpobench/container/benchmarks/ml/xgboost_benchmark.py @@ -13,6 +13,12 @@ def __init__(self, **kwargs): kwargs['latest'] = kwargs.get('container_tag', '0.0.1') super(XGBoostBenchmark, self).__init__(**kwargs) +class XGBoostBenchmarkMO(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'XGBoostBenchmarkMO') + kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') + kwargs['latest'] = kwargs.get('container_tag', '0.0.2') + super(XGBoostBenchmarkMO, self).__init__(**kwargs) class XGBoostBenchmarkBB(AbstractBenchmarkClient): def __init__(self, **kwargs): diff --git a/hpobench/container/benchmarks/ml/xgboost_benchmark_old.py b/hpobench/container/benchmarks/ml/xgboost_benchmark_old.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/mo/__init__.py b/hpobench/container/benchmarks/mo/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/mo/adult_benchmark.py b/hpobench/container/benchmarks/mo/adult_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/mo/cnn_benchmark.py b/hpobench/container/benchmarks/mo/cnn_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/nas/__init__.py b/hpobench/container/benchmarks/nas/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/nas/jahs_benchmarks.py b/hpobench/container/benchmarks/nas/jahs_benchmarks.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/nas/nasbench_101.py b/hpobench/container/benchmarks/nas/nasbench_101.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/nas/nasbench_1shot1.py b/hpobench/container/benchmarks/nas/nasbench_1shot1.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/nas/nasbench_201.py b/hpobench/container/benchmarks/nas/nasbench_201.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/nas/tabular_benchmarks.py b/hpobench/container/benchmarks/nas/tabular_benchmarks.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/od/__init__.py b/hpobench/container/benchmarks/od/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/od/od_benchmarks.py b/hpobench/container/benchmarks/od/od_benchmarks.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/rl/__init__.py b/hpobench/container/benchmarks/rl/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/rl/cartpole.py b/hpobench/container/benchmarks/rl/cartpole.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/rl/learna_benchmark.py b/hpobench/container/benchmarks/rl/learna_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/surrogates/__init__.py b/hpobench/container/benchmarks/surrogates/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/surrogates/paramnet_benchmark.py b/hpobench/container/benchmarks/surrogates/paramnet_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/surrogates/svm_benchmark.py b/hpobench/container/benchmarks/surrogates/svm_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/surrogates/yahpo_gym.py b/hpobench/container/benchmarks/surrogates/yahpo_gym.py old mode 100644 new mode 100755 diff --git a/hpobench/container/client_abstract_benchmark.py b/hpobench/container/client_abstract_benchmark.py old mode 100644 new mode 100755 index d2963c00..705c4893 --- a/hpobench/container/client_abstract_benchmark.py +++ b/hpobench/container/client_abstract_benchmark.py @@ -165,7 +165,10 @@ def _parse_fidelities(self, fidelity: Union[CS.Configuration, Dict, None] = None fidelity = fidelity else: raise ValueError(f'Type of fidelity not understood: {type(fidelity)}') + print("fidelity",type(fidelity)) + #print("fidelity type", type(fidelity['nepochs'])) f_str = json.dumps(fidelity, indent=None, cls=BenchmarkEncoder) + print("fstr",f_str) return f_str def load_benchmark(self, benchmark_name: str, container_name: str, container_source: Optional[str] = None, @@ -389,9 +392,16 @@ def objective_function(self, configuration: Union[CS.Configuration, Dict], ------- Dict """ + logger.info(f"fidelity:{fidelity}") + + #logger.info(f"type fidelity:{type(fidelity['nepochs'])}") + logger.info(f"config:{configuration}") + #fidelity = fidelity['nepochs'] c_str = self._parse_configuration(configuration) f_str = self._parse_fidelities(fidelity) kwargs_str = self._parse_kwargs(rng, **kwargs) + print("c_str",c_str) + print("fidelity f_str",f_str) json_str = self.benchmark.objective_function(c_str, f_str, kwargs_str) return json.loads(json_str, cls=BenchmarkDecoder) diff --git a/hpobench/container/recipes/Singularity.template b/hpobench/container/recipes/Singularity.template old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/ml/Singularity.PyBNN b/hpobench/container/recipes/ml/Singularity.PyBNN old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/ml/Singularity.SupportVectorMachine b/hpobench/container/recipes/ml/Singularity.SupportVectorMachine old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/ml/Singularity.XGBoostBenchmark b/hpobench/container/recipes/ml/Singularity.XGBoostBenchmark old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/ml/Singularity.ml_mmfb b/hpobench/container/recipes/ml/Singularity.ml_mmfb old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/ml/Singularity.ml_tabular_benchmarks b/hpobench/container/recipes/ml/Singularity.ml_tabular_benchmarks old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/mo/Singularity.AdultBenchmark b/hpobench/container/recipes/mo/Singularity.AdultBenchmark old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/mo/Singularity.CNNBenchmark b/hpobench/container/recipes/mo/Singularity.CNNBenchmark old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/nas/Singularity.TabularBenchmarks b/hpobench/container/recipes/nas/Singularity.TabularBenchmarks old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/nas/Singularity.nasbench_101 b/hpobench/container/recipes/nas/Singularity.nasbench_101 old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/nas/Singularity.nasbench_1shot1 b/hpobench/container/recipes/nas/Singularity.nasbench_1shot1 old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/nas/Singularity.nasbench_201 b/hpobench/container/recipes/nas/Singularity.nasbench_201 old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/od/Singularity.ODBenchmarks b/hpobench/container/recipes/od/Singularity.ODBenchmarks old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/od/Singularity.ODKernelDensityEstimation b/hpobench/container/recipes/od/Singularity.ODKernelDensityEstimation old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/rl/Singularity.Cartpole b/hpobench/container/recipes/rl/Singularity.Cartpole old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/rl/Singularity.learnaBenchmark b/hpobench/container/recipes/rl/Singularity.learnaBenchmark old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/surrogates/Singularity.ParamnetBenchmark b/hpobench/container/recipes/surrogates/Singularity.ParamnetBenchmark old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/surrogates/Singularity.SupportVectorMachine b/hpobench/container/recipes/surrogates/Singularity.SupportVectorMachine old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/surrogates/Singularity.YAHPOGymBenchmark b/hpobench/container/recipes/surrogates/Singularity.YAHPOGymBenchmark old mode 100644 new mode 100755 diff --git a/hpobench/container/server_abstract_benchmark.py b/hpobench/container/server_abstract_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/__init__.py b/hpobench/dependencies/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/ml/__init__.py b/hpobench/dependencies/ml/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/ml/data_manager.py b/hpobench/dependencies/ml/data_manager.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/ml/ml_benchmark_template.py b/hpobench/dependencies/ml/ml_benchmark_template.py old mode 100644 new mode 100755 index 7cef515f..9ed139ba --- a/hpobench/dependencies/ml/ml_benchmark_template.py +++ b/hpobench/dependencies/ml/ml_benchmark_template.py @@ -11,6 +11,8 @@ from hpobench.abstract_benchmark import AbstractBenchmark from hpobench.dependencies.ml.data_manager import OpenMLDataManager from hpobench.util.rng_helper import get_rng +from hpobench.abstract_benchmark import AbstractMultiObjectiveBenchmark, AbstractSingleObjectiveBenchmark +from typing import Union, Dict, List, Text, Tuple metrics = dict( acc=accuracy_score, @@ -27,7 +29,7 @@ ) -class MLBenchmark(AbstractBenchmark): +class _MLBenchmarkBase(): _issue_tasks = [3917, 3945] def __init__( @@ -38,7 +40,7 @@ def __init__( data_path: Union[str, Path, None] = None, global_seed: int = 1 ): - super(MLBenchmark, self).__init__(rng=rng) + super(_MLBenchmarkBase, self).__init__(rng=rng) if isinstance(rng, int): self.seed = rng @@ -197,16 +199,38 @@ def _train_objective(self, score_cost[k] = time.time() - _start train_loss = 1 - scores["acc"] return model, model_fit_time, train_loss, scores, score_cost - - # pylint: disable=arguments-differ - @AbstractBenchmark.check_parameters - def objective_function(self, - configuration: Union[CS.Configuration, Dict], - fidelity: Union[CS.Configuration, Dict, None] = None, - shuffle: bool = False, - rng: Union[np.random.RandomState, int, None] = None, - **kwargs) -> Dict: + + @AbstractMultiObjectiveBenchmark.check_parameters + def _mo_objective_function(self, + configuration: Union[CS.Configuration, Dict], + fidelity: Union[CS.Configuration, Dict, None] = None, + shuffle: bool = False, + rng: Union[np.random.RandomState, int, None] = None, + **kwargs) -> Dict: """Function that evaluates a 'config' on a 'fidelity' on the validation set + + Parameters + ---------- + configuration : Dict, CS.Configuration + fidelity: Dict, None + Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None. + rng : np.random.RandomState, int, None + Random seed to use in the benchmark. + + To prevent overfitting on a single seed, it is possible to pass a + parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. + If this parameter is not given, the default random state is used. + kwargs + + Returns + ------- + Dict - + function_value : + ToDo: add return values + cost : runtime + info : Dict + fidelity : used fidelities in this evaluation + """ model, model_fit_time, train_loss, train_scores, train_score_cost = self._train_objective( configuration, fidelity, shuffle, rng, evaluation="val" @@ -218,6 +242,8 @@ def objective_function(self, val_scores[k] = v(model, self.valid_X, self.valid_y) val_score_cost[k] = time.time() - _start val_loss = 1 - val_scores["acc"] + val_sensitivity = (val_scores['f1'] * val_scores['precision']) / (2 * val_scores['precision'] - val_scores['f1'] ) + val_scores['false_negative_rate'] = 1 - val_sensitivity test_scores = dict() test_score_cost = dict() @@ -227,6 +253,9 @@ def objective_function(self, test_score_cost[k] = time.time() - _start test_loss = 1 - test_scores["acc"] + test_sensitivity = (test_scores['f1'] * test_scores['precision']) / (2 * test_scores['precision'] - test_scores['f1'] ) + test_scores['false_negative_rate'] = 1 - test_sensitivity + info = { 'train_loss': train_loss, 'val_loss': val_loss, @@ -243,15 +272,22 @@ def objective_function(self, 'config': configuration, } - return { - 'function_value': info['val_loss'], + result_dict = { + 'function_value': { + # The original benchmark returned the accuracy with range [0, 100]. + # We cast it to a minimization problem with range [0-1] to have a more standardized return value. + 'val_loss': val_loss, + 'sensitivity': val_scores['false_negative_rate'], + + }, 'cost': model_fit_time + info['val_costs']['acc'], 'info': info } - # pylint: disable=arguments-differ - @AbstractBenchmark.check_parameters - def objective_function_test(self, + return result_dict + + @AbstractMultiObjectiveBenchmark.check_parameters + def _mo_objective_function_test(self, configuration: Union[CS.Configuration, Dict], fidelity: Union[CS.Configuration, Dict, None] = None, shuffle: bool = False, @@ -270,6 +306,9 @@ def objective_function_test(self, test_score_cost[k] = time.time() - _start test_loss = 1 - test_scores["acc"] + test_sensitivity = (test_scores['f1'] * test_scores['precision']) / (2 * test_scores['precision'] - test_scores['f1'] ) + test_scores['false_negative_rate'] = 1 - test_sensitivity + info = { 'train_loss': train_loss, 'val_loss': None, @@ -287,7 +326,100 @@ def objective_function_test(self, } return { - 'function_value': float(info['test_loss']), + 'function_value': { + # The original benchmark returned the accuracy with range [0, 100]. + # We cast it to a minimization problem with range [0-1] to have a more standardized return value. + 'test_loss': float(info['test_loss']), + # 'f1': test_scores['f1'], + # 'precision': test_scores['precision'], + # 'balanced_accuracy': test_scores['bal_acc'], + 'sensitivity': test_scores['false_negative_rate'], + }, 'cost': float(model_fit_time + info['test_costs']['acc']), 'info': info } + + +class MLBenchmark(_MLBenchmarkBase, AbstractSingleObjectiveBenchmark): + + + @AbstractSingleObjectiveBenchmark.check_parameters + def objective_function(self, + configuration: Union[CS.Configuration, Dict], + fidelity: Union[CS.Configuration, Dict, None] = None, + shuffle: bool = False, + rng: Union[np.random.RandomState, int, None] = None, + **kwargs) -> Dict: + """Function that evaluates a 'config' on a 'fidelity' on the validation set + + Parameters + ---------- + configuration : Dict, CS.Configuration + fidelity: Dict, None + Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None. + rng : np.random.RandomState, int, None + Random seed to use in the benchmark. + + To prevent overfitting on a single seed, it is possible to pass a + parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. + If this parameter is not given, the default random state is used. + kwargs + + Returns + ------- + Dict - + function_value : + ToDo: add return values + cost : runtime + info : Dict + fidelity : used fidelities in this evaluation + + """ + results = self._mo_objective_function( + configuration=configuration, fidelity=fidelity, rng=rng, **kwargs + ) + results['function_value'] = results['function_value']['val_loss'] + return results + + @AbstractSingleObjectiveBenchmark.check_parameters + def objective_function_test(self, + configuration: Union[CS.Configuration, Dict], + fidelity: Union[CS.Configuration, Dict, None] = None, + shuffle: bool = False, + rng: Union[np.random.RandomState, int, None] = None, + **kwargs) -> Dict: + """Function that evaluates a 'config' on a 'fidelity' on the test set + """ + + results = self._mo_objective_function_test( + configuration=configuration, fidelity=fidelity, rng=rng, **kwargs + ) + results['function_value'] = results['function_value']['test_loss'] + return results + +class MO_MLBenchmark(_MLBenchmarkBase, AbstractMultiObjectiveBenchmark): + + @AbstractMultiObjectiveBenchmark.check_parameters + def objective_function(self, + configuration: Union[CS.Configuration, Dict], + fidelity: Union[CS.Configuration, Dict, None] = None, + shuffle: bool = False, + rng: Union[np.random.RandomState, int, None] = None, + **kwargs) -> Dict: + return self._mo_objective_function(configuration=configuration, fidelity=fidelity,shuffle=shuffle, rng=rng, + **kwargs) + + @AbstractMultiObjectiveBenchmark.check_parameters + def objective_function_test(self, + configuration: Union[CS.Configuration, Dict], + fidelity: Union[CS.Configuration, Dict, None] = None, + shuffle: bool = False, + rng: Union[np.random.RandomState, int, None] = None, + **kwargs) -> Dict: + return self._mo_objective_function_test(configuration=configuration, fidelity=fidelity, shuffle=shuffle, rng=rng, + **kwargs) + @staticmethod + def get_objective_names() -> List[str]: + return ['val_loss', 'bal_acc', 'f1', 'precision', 'sensitivity'] + + diff --git a/hpobench/dependencies/mo/__init__.py b/hpobench/dependencies/mo/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/mo/fairness_metrics.py b/hpobench/dependencies/mo/fairness_metrics.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/mo/scalar.py b/hpobench/dependencies/mo/scalar.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/__init__.py b/hpobench/dependencies/od/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/backbones/__init__.py b/hpobench/dependencies/od/backbones/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/backbones/mlp.py b/hpobench/dependencies/od/backbones/mlp.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/callbacks/__init__.py b/hpobench/dependencies/od/callbacks/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/callbacks/checkpoint_saver.py b/hpobench/dependencies/od/callbacks/checkpoint_saver.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/callbacks/earlystopping.py b/hpobench/dependencies/od/callbacks/earlystopping.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/data_manager.py b/hpobench/dependencies/od/data_manager.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/models/__init__.py b/hpobench/dependencies/od/models/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/models/autoencoder.py b/hpobench/dependencies/od/models/autoencoder.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/traditional_benchmark.py b/hpobench/dependencies/od/traditional_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/utils/__init__.py b/hpobench/dependencies/od/utils/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/utils/activations.py b/hpobench/dependencies/od/utils/activations.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/utils/scaler.py b/hpobench/dependencies/od/utils/scaler.py old mode 100644 new mode 100755 diff --git a/hpobench/util/__init__.py b/hpobench/util/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/util/clean_up_script.py b/hpobench/util/clean_up_script.py old mode 100644 new mode 100755 diff --git a/hpobench/util/container_utils.py b/hpobench/util/container_utils.py old mode 100644 new mode 100755 diff --git a/hpobench/util/data_manager.py b/hpobench/util/data_manager.py old mode 100644 new mode 100755 diff --git a/hpobench/util/dependencies.py b/hpobench/util/dependencies.py old mode 100644 new mode 100755 diff --git a/hpobench/util/example_utils.py b/hpobench/util/example_utils.py old mode 100644 new mode 100755 diff --git a/hpobench/util/openml_data_manager.py b/hpobench/util/openml_data_manager.py old mode 100644 new mode 100755 index 6af5709f..093f417f --- a/hpobench/util/openml_data_manager.py +++ b/hpobench/util/openml_data_manager.py @@ -69,6 +69,7 @@ def get_openmlcc18_taskids(): def _load_data(task_id: int): """ Helper-function to load the data from the OpenML website. """ + print("openml task id", task_id) task = openml.tasks.get_task(task_id) try: diff --git a/hpobench/util/rng_helper.py b/hpobench/util/rng_helper.py old mode 100644 new mode 100755 diff --git a/hpobench/util/test_utils.py b/hpobench/util/test_utils.py old mode 100644 new mode 100755 diff --git a/requirements.txt b/requirements.txt old mode 100644 new mode 100755 diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 diff --git a/tests/__init__.py b/tests/__init__.py old mode 100644 new mode 100755 diff --git a/tests/test_abstract_benchmark.py b/tests/test_abstract_benchmark.py old mode 100644 new mode 100755 diff --git a/tests/test_adult.py b/tests/test_adult.py old mode 100644 new mode 100755 index b52c37ed..0b2a7a0b --- a/tests/test_adult.py +++ b/tests/test_adult.py @@ -6,9 +6,17 @@ def test_adult_benchmark(): from hpobench.container.benchmarks.mo.adult_benchmark import AdultBenchmark + from hpobench.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark + from hpobench.benchmarks.mo.adult_benchmark import AdultBenchmark # Check Seeding benchmark = AdultBenchmark(rng=0) + print("got adult bench") + print("getting ml bench") + b = XGBoostBenchmark(task_id=167149) + print("b:",b) + + print("got ml bench") cs = benchmark.get_configuration_space(seed=0) cfg_1 = cs.sample_configuration() @@ -35,3 +43,4 @@ def test_adult_benchmark(): result_1 = benchmark.objective_function_test(test_config, rng=1, fidelity={'budget': 3}) assert 1 - result_1['function_value']['misclassification_rate'] == pytest.approx(0.76377, rel=0.001) assert 1 - result_1['function_value']['misclassification_rate'] == result_1['info']['test_accuracy'] +test_adult_benchmark() \ No newline at end of file diff --git a/tests/test_check_configuration.py b/tests/test_check_configuration.py old mode 100644 new mode 100755 diff --git a/tests/test_container_availbable.py b/tests/test_container_availbable.py old mode 100644 new mode 100755 diff --git a/tests/test_data_manager.py b/tests/test_data_manager.py old mode 100644 new mode 100755 diff --git a/tests/test_hpobenchconfig.py b/tests/test_hpobenchconfig.py old mode 100644 new mode 100755 diff --git a/tests/test_mo_cnn.py b/tests/test_mo_cnn.py old mode 100644 new mode 100755 diff --git a/tests/test_nasbench_101.py b/tests/test_nasbench_101.py old mode 100644 new mode 100755 diff --git a/tests/test_nasbench_201.py b/tests/test_nasbench_201.py old mode 100644 new mode 100755 diff --git a/tests/test_od.py b/tests/test_od.py old mode 100644 new mode 100755 diff --git a/tests/test_openml_datamanager.py b/tests/test_openml_datamanager.py old mode 100644 new mode 100755 diff --git a/tests/test_paramnet.py b/tests/test_paramnet.py old mode 100644 new mode 100755 diff --git a/tests/test_pybnn.py b/tests/test_pybnn.py old mode 100644 new mode 100755 diff --git a/tests/test_server.py b/tests/test_server.py old mode 100644 new mode 100755 diff --git a/tests/test_svm.py b/tests/test_svm.py old mode 100644 new mode 100755 index c3acf007..444f6dd5 --- a/tests/test_svm.py +++ b/tests/test_svm.py @@ -1,6 +1,6 @@ import pytest -from hpobench.container.benchmarks.ml.svm_benchmark_old import SupportVectorMachine +from hpobench.benchmarks.ml.svm_benchmark import SVMBenchmark from hpobench.util.openml_data_manager import get_openmlcc18_taskids task_ids = get_openmlcc18_taskids() @@ -8,9 +8,8 @@ import logging logging.basicConfig(level=logging.DEBUG) - def test_svm_init(): - benchmark = SupportVectorMachine(task_id=task_ids[0]) + benchmark = SVMBenchmark(task_id=task_ids[0]) fs = benchmark.get_fidelity_space(seed=0) fidelity = fs.sample_configuration().get_dictionary() @@ -34,3 +33,29 @@ def test_svm_init(): result = benchmark.objective_function_test(configuration=config) assert result['function_value'] == pytest.approx(0.4648, abs=0.1) assert result['cost'] is not None + +# def test_svm_init(): +# benchmark = SupportVectorMachine(task_id=task_ids[0]) + +# fs = benchmark.get_fidelity_space(seed=0) +# fidelity = fs.sample_configuration().get_dictionary() +# assert fidelity['dataset_fraction'] == pytest.approx(0.54881, abs=0.001) + +# meta = benchmark.get_meta_information() +# assert meta is not None + +# cs = benchmark.get_configuration_space(seed=0) +# config = cs.sample_configuration().get_dictionary() +# assert config['C'] == pytest.approx(0.9762, abs=0.001) +# assert config['gamma'] == pytest.approx(4.3037, abs=0.001) + +# result = benchmark.objective_function(configuration=config, fidelity=fidelity) +# assert result['function_value'] == pytest.approx(0.4837, abs=0.1) +# assert result['cost'] is not None + +# with pytest.raises(AssertionError): +# result = benchmark.objective_function_test(configuration=config, fidelity=fidelity) + +# result = benchmark.objective_function_test(configuration=config) +# assert result['function_value'] == pytest.approx(0.4648, abs=0.1) +# assert result['cost'] is not None diff --git a/tests/test_tabular_benchmarks.py b/tests/test_tabular_benchmarks.py old mode 100644 new mode 100755 diff --git a/tests/test_utils.py b/tests/test_utils.py old mode 100644 new mode 100755 diff --git a/tests/test_whitebox.py b/tests/test_whitebox.py old mode 100644 new mode 100755 diff --git a/tests/test_yahpo.py b/tests/test_yahpo.py old mode 100644 new mode 100755 From d8fcfeb1236a03722cabe2146e637b883dc6ecfc Mon Sep 17 00:00:00 2001 From: ayushi-3536 Date: Tue, 14 Mar 2023 23:32:21 +0100 Subject: [PATCH 02/17] added compatibility for mo in openml --- .../container/benchmarks/ml/xgboost_benchmark.py | 7 +++++-- hpobench/container/client_abstract_benchmark.py | 3 ++- hpobench/dependencies/ml/ml_benchmark_template.py | 12 ++++++------ 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/hpobench/container/benchmarks/ml/xgboost_benchmark.py b/hpobench/container/benchmarks/ml/xgboost_benchmark.py index 38c85c8e..88d24258 100755 --- a/hpobench/container/benchmarks/ml/xgboost_benchmark.py +++ b/hpobench/container/benchmarks/ml/xgboost_benchmark.py @@ -16,10 +16,13 @@ def __init__(self, **kwargs): class XGBoostBenchmarkMO(AbstractBenchmarkClient): def __init__(self, **kwargs): kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'XGBoostBenchmarkMO') - kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') - kwargs['latest'] = kwargs.get('container_tag', '0.0.2') + kwargs['container_name'] = kwargs.get('container_name', 'mo_ml_mmfb') + kwargs['container_source'] = 'oras://gitlab.tf.uni-freiburg.de:5050/sharmaa/hpobench-registry' + kwargs['container_tag'] = '0.0.2' + kwargs['latest'] = '0.0.2' #kwargs.get('container_tag', '0.0.2') super(XGBoostBenchmarkMO, self).__init__(**kwargs) + class XGBoostBenchmarkBB(AbstractBenchmarkClient): def __init__(self, **kwargs): kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'XGBoostBenchmarkBB') diff --git a/hpobench/container/client_abstract_benchmark.py b/hpobench/container/client_abstract_benchmark.py index 705c4893..163a3eff 100755 --- a/hpobench/container/client_abstract_benchmark.py +++ b/hpobench/container/client_abstract_benchmark.py @@ -225,7 +225,8 @@ def download_container(container_dir, container_name, container_source, containe # Currently, we can't see the available container tags on gitlab. Therefore, we create for each # "tag" a new entry in the registry. This might change in the future. But as long as we don't have # a fix for this, we need to map the container tag differently. - if container_source.startswith('oras://gitlab.tf.uni-freiburg.de:5050/muelleph/hpobench-registry'): + if (container_source.startswith('oras://gitlab.tf.uni-freiburg.de:5050/muelleph/hpobench-registry')) or \ + (container_source.startswith('oras://gitlab.tf.uni-freiburg.de:5050/sharmaa/hpobench-registry')): cmd += f'{container_source}/{container_name.lower()}/{container_tag}:latest' else: cmd += f'{container_source}/{container_name.lower()}:{container_tag}' diff --git a/hpobench/dependencies/ml/ml_benchmark_template.py b/hpobench/dependencies/ml/ml_benchmark_template.py index 9ed139ba..748a7bbc 100755 --- a/hpobench/dependencies/ml/ml_benchmark_template.py +++ b/hpobench/dependencies/ml/ml_benchmark_template.py @@ -243,7 +243,7 @@ def _mo_objective_function(self, val_score_cost[k] = time.time() - _start val_loss = 1 - val_scores["acc"] val_sensitivity = (val_scores['f1'] * val_scores['precision']) / (2 * val_scores['precision'] - val_scores['f1'] ) - val_scores['false_negative_rate'] = 1 - val_sensitivity + val_scores['fnr'] = 1 - val_sensitivity test_scores = dict() test_score_cost = dict() @@ -254,7 +254,7 @@ def _mo_objective_function(self, test_loss = 1 - test_scores["acc"] test_sensitivity = (test_scores['f1'] * test_scores['precision']) / (2 * test_scores['precision'] - test_scores['f1'] ) - test_scores['false_negative_rate'] = 1 - test_sensitivity + test_scores['fnr'] = 1 - test_sensitivity info = { 'train_loss': train_loss, @@ -277,7 +277,7 @@ def _mo_objective_function(self, # The original benchmark returned the accuracy with range [0, 100]. # We cast it to a minimization problem with range [0-1] to have a more standardized return value. 'val_loss': val_loss, - 'sensitivity': val_scores['false_negative_rate'], + 'fnr': val_scores['fnr'], }, 'cost': model_fit_time + info['val_costs']['acc'], @@ -307,7 +307,7 @@ def _mo_objective_function_test(self, test_loss = 1 - test_scores["acc"] test_sensitivity = (test_scores['f1'] * test_scores['precision']) / (2 * test_scores['precision'] - test_scores['f1'] ) - test_scores['false_negative_rate'] = 1 - test_sensitivity + test_scores['fnr'] = 1 - test_sensitivity info = { 'train_loss': train_loss, @@ -333,7 +333,7 @@ def _mo_objective_function_test(self, # 'f1': test_scores['f1'], # 'precision': test_scores['precision'], # 'balanced_accuracy': test_scores['bal_acc'], - 'sensitivity': test_scores['false_negative_rate'], + 'fnr': test_scores['fnr'], }, 'cost': float(model_fit_time + info['test_costs']['acc']), 'info': info @@ -420,6 +420,6 @@ def objective_function_test(self, **kwargs) @staticmethod def get_objective_names() -> List[str]: - return ['val_loss', 'bal_acc', 'f1', 'precision', 'sensitivity'] + return ['val_loss', 'fnr', bal_acc', 'f1', 'precision'] From 8055fdc78425bb047f4d31f19ca13b6664cdc157 Mon Sep 17 00:00:00 2001 From: ayushi-3536 Date: Wed, 15 Mar 2023 00:38:39 +0100 Subject: [PATCH 03/17] adapt openml for mo --- hpobench/benchmarks/ml/__init__.py | 26 +-- hpobench/benchmarks/ml/nn_benchmark.py | 34 +++- hpobench/benchmarks/ml/rf_benchmark.py | 45 +++-- hpobench/benchmarks/ml/svm_benchmark.py | 45 +++-- hpobench/benchmarks/ml/xgboost_benchmark.py | 35 +++- .../benchmarks/ml/histgb_benchmark.py | 8 +- .../container/benchmarks/ml/lr_benchmark.py | 9 +- .../container/benchmarks/ml/nn_benchmark.py | 8 +- .../container/benchmarks/ml/rf_benchmark.py | 11 +- .../container/benchmarks/ml/svm_benchmark.py | 9 +- .../benchmarks/ml/xgboost_benchmark.py | 11 +- .../container/client_abstract_benchmark.py | 3 +- .../dependencies/ml/ml_benchmark_template.py | 165 ++++++++++++++++-- 13 files changed, 336 insertions(+), 73 deletions(-) diff --git a/hpobench/benchmarks/ml/__init__.py b/hpobench/benchmarks/ml/__init__.py index 64e399cd..fcdac8d6 100644 --- a/hpobench/benchmarks/ml/__init__.py +++ b/hpobench/benchmarks/ml/__init__.py @@ -1,22 +1,22 @@ -from hpobench.benchmarks.ml.histgb_benchmark import HistGBBenchmark, HistGBBenchmarkBB, HistGBBenchmarkMF -from hpobench.benchmarks.ml.lr_benchmark import LRBenchmark, LRBenchmarkBB, LRBenchmarkMF -from hpobench.benchmarks.ml.nn_benchmark import NNBenchmark, NNBenchmarkBB, NNBenchmarkMF +from hpobench.benchmarks.ml.histgb_benchmark import HistGBBenchmark, HistGBBenchmarkBB, HistGBBenchmarkMF, HistGBBenchmarkMO +from hpobench.benchmarks.ml.lr_benchmark import LRBenchmark, LRBenchmarkBB, LRBenchmarkMF, LRBenchmarkMO +from hpobench.benchmarks.ml.nn_benchmark import NNBenchmark, NNBenchmarkBB, NNBenchmarkMF, NNBenchmarkMO from hpobench.benchmarks.ml.rf_benchmark import RandomForestBenchmark, RandomForestBenchmarkBB, \ - RandomForestBenchmarkMF -from hpobench.benchmarks.ml.svm_benchmark import SVMBenchmark, SVMBenchmarkBB, SVMBenchmarkMF + RandomForestBenchmarkMF, RandomForestBenchmarkMO +from hpobench.benchmarks.ml.svm_benchmark import SVMBenchmark, SVMBenchmarkBB, SVMBenchmarkMF, SVMBenchmarkMO from hpobench.benchmarks.ml.tabular_benchmark import TabularBenchmark try: - from hpobench.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark, XGBoostBenchmarkBB, XGBoostBenchmarkMF + from hpobench.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark, XGBoostBenchmarkBB, XGBoostBenchmarkMF, XGBoostBenchmarkMO except ImportError: pass -__all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF', - 'LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF', - 'NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF', - 'RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF', - 'SVMBenchmark', 'SVMBenchmarkBB', 'SVMBenchmarkMF', +__all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF', 'HistGBBenchmarkMO', + 'LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF', 'LRBenchmarkMO', + 'NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF', 'NNBenchmarkMO', + 'RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF', 'RandomForestBenchmarkMO', + 'SVMBenchmark', 'SVMBenchmarkBB', 'SVMBenchmarkMF', 'SVMBenchmarkMO', 'TabularBenchmark', - 'XGBoostBenchmark', 'XGBoostBenchmarkBB', 'XGBoostBenchmarkMF', - ] + 'XGBoostBenchmark', 'XGBoostBenchmarkBB', 'XGBoostBenchmarkMF', 'XGBoostBenchmarkMO', + ] \ No newline at end of file diff --git a/hpobench/benchmarks/ml/nn_benchmark.py b/hpobench/benchmarks/ml/nn_benchmark.py index 06634661..a16d967b 100644 --- a/hpobench/benchmarks/ml/nn_benchmark.py +++ b/hpobench/benchmarks/ml/nn_benchmark.py @@ -14,18 +14,28 @@ from ConfigSpace.hyperparameters import Hyperparameter from sklearn.neural_network import MLPClassifier -from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark +from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark, MO_MLBenchmark -__version__ = '0.0.1' +''' +Changelog: +========== +0.0.2: +* Add the multiobjective version of this benchmark by returning val loss, precision, f1 and balanced accuracy + +0.0.1: +* First implementation +''' + +__version__ = '0.0.2' -class NNBenchmark(MLBenchmark): +class _NNBenchmarkBase: def __init__(self, task_id: int, rng: Union[np.random.RandomState, int, None] = None, valid_size: float = 0.33, data_path: Union[str, None] = None): - super(NNBenchmark, self).__init__(task_id, rng, valid_size, data_path) + super(_NNBenchmarkBase, self).__init__(task_id, rng, valid_size, data_path) @staticmethod def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -109,6 +119,13 @@ def init_model(self, config: Union[CS.Configuration, Dict], ) return model +class NNBenchmark(_NNBenchmarkBase, MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(NNBenchmark, self).__init__(task_id, rng, valid_size, data_path) class NNBenchmarkBB(NNBenchmark): def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -129,5 +146,12 @@ def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationS ) return fidelity_space +class NNBenchmarkMO(_NNBenchmarkBase, MO_MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(NNBenchmark, self).__init__(task_id, rng, valid_size, data_path) -__all__ = ['NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF'] +__all__ = ['NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF', 'NNBenchmarkMO'] \ No newline at end of file diff --git a/hpobench/benchmarks/ml/rf_benchmark.py b/hpobench/benchmarks/ml/rf_benchmark.py index 596f03b6..68004e02 100644 --- a/hpobench/benchmarks/ml/rf_benchmark.py +++ b/hpobench/benchmarks/ml/rf_benchmark.py @@ -1,11 +1,3 @@ -""" -Changelog: -========== - -0.0.1: -* First implementation of the RF Benchmarks. -""" - from copy import deepcopy from typing import Union, Tuple, Dict @@ -14,18 +6,27 @@ from ConfigSpace.hyperparameters import Hyperparameter from sklearn.ensemble import RandomForestClassifier -from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark +from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark, MO_MLBenchmark + +''' +Changelog: +========== +0.0.2: +* Add the multiobjective version of this benchmark by returning val loss, precision, f1 and balanced accuracy -__version__ = '0.0.1' +0.0.1: +* First implementation +''' +__version__ = '0.0.2' -class RandomForestBenchmark(MLBenchmark): +class _RandomForestBenchmarkBase: def __init__(self, task_id: int, rng: Union[np.random.RandomState, int, None] = None, valid_size: float = 0.33, data_path: Union[str, None] = None): - super(RandomForestBenchmark, self).__init__(task_id, rng, valid_size, data_path) + super(_RandomForestBenchmarkBase, self).__init__(task_id, rng, valid_size, data_path) @staticmethod def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -103,6 +104,13 @@ def init_model(self, config: Union[CS.Configuration, Dict], ) return model +class RandomForestBenchmark(_RandomForestBenchmarkBase, MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(RandomForestBenchmark, self).__init__(task_id, rng, valid_size, data_path) class RandomForestBenchmarkBB(RandomForestBenchmark): def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -122,6 +130,17 @@ def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationS RandomForestBenchmark._get_fidelity_choices(n_estimators_choice='variable', subsample_choice='fixed') ) return fidelity_space + +class RandomForestBenchmarkMO(_RandomForestBenchmarkBase, MO_MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(RandomForestBenchmarkMO, self).__init__(task_id, rng, valid_size, data_path) + + -__all__ = ['RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF'] +__all__ = ['RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF', + 'RandomForestBenchmarkMO'] \ No newline at end of file diff --git a/hpobench/benchmarks/ml/svm_benchmark.py b/hpobench/benchmarks/ml/svm_benchmark.py index 9462442f..48f62765 100644 --- a/hpobench/benchmarks/ml/svm_benchmark.py +++ b/hpobench/benchmarks/ml/svm_benchmark.py @@ -1,11 +1,3 @@ -""" -Changelog: -========== - -0.0.1: -* First implementation of the new SVM Benchmarks. -""" - from typing import Union, Dict import ConfigSpace as CS @@ -13,18 +5,27 @@ from ConfigSpace.hyperparameters import Hyperparameter from sklearn.svm import SVC -from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark +from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark, MO_MLBenchmark +''' +Changelog: +========== +0.0.2: +* Add the multiobjective version of this benchmark by returning val loss, precision, f1 and balanced accuracy + +0.0.1: +* First implementation +''' -__version__ = '0.0.1' +__version__ = '0.0.2' -class SVMBenchmark(MLBenchmark): +class _SVMBenchmarkBase: def __init__(self, task_id: int, rng: Union[np.random.RandomState, int, None] = None, valid_size: float = 0.33, data_path: Union[str, None] = None): - super(SVMBenchmark, self).__init__(task_id, rng, valid_size, data_path) + super(_SVMBenchmarkBase, self).__init__(task_id, rng, valid_size, data_path) self.cache_size = 200 @@ -81,6 +82,14 @@ def init_model(self, config: Union[CS.Configuration, Dict], ) return model +class SVMBenchmark(_SVMBenchmarkBase, MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(SVMBenchmark, self).__init__(task_id, rng, valid_size, data_path) + class SVMBenchmarkBB(SVMBenchmark): def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -90,9 +99,19 @@ def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationS SVMBenchmark._get_fidelity_choices(subsample_choice='fixed') ) return fidelity_space + +class SVMBenchmarkMO(_SVMBenchmarkBase, MO_MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(SVMBenchmarkMO, self).__init__(task_id, rng, valid_size, data_path) + + # To keep the parity of the the overall design SVMBenchmarkMF = SVMBenchmark -__all__ = ['SVMBenchmark', 'SVMBenchmarkMF', 'SVMBenchmarkBB'] +__all__ = ['SVMBenchmark', 'SVMBenchmarkMF', 'SVMBenchmarkBB', 'SVMBenchmarkMO'] \ No newline at end of file diff --git a/hpobench/benchmarks/ml/xgboost_benchmark.py b/hpobench/benchmarks/ml/xgboost_benchmark.py index ae554628..27509adb 100644 --- a/hpobench/benchmarks/ml/xgboost_benchmark.py +++ b/hpobench/benchmarks/ml/xgboost_benchmark.py @@ -12,18 +12,27 @@ import xgboost as xgb from ConfigSpace.hyperparameters import Hyperparameter -from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark +from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark, MO_MLBenchmark -__version__ = '0.0.1' +''' +Changelog: +========== +0.0.2: +* Add the multiobjective version of this benchmark by returning val loss, precision, f1 and balanced accuracy +0.0.1: +* First implementation +''' +__version__ = '0.0.2' -class XGBoostBenchmark(MLBenchmark): + +class _XGBoostBenchmarkBase: def __init__(self, task_id: int, rng: Union[np.random.RandomState, int, None] = None, valid_size: float = 0.33, data_path: Union[str, None] = None): - super(XGBoostBenchmark, self).__init__(task_id, rng, valid_size, data_path) + super(_XGBoostBenchmarkBase, self).__init__(task_id, rng, valid_size, data_path) @staticmethod def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -108,6 +117,13 @@ def init_model(self, ) return model +class XGBoostBenchmark(_XGBoostBenchmarkBase, MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(XGBoostBenchmark, self).__init__(task_id, rng, valid_size, data_path) class XGBoostBenchmarkBB(XGBoostBenchmark): def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -127,6 +143,15 @@ def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationS XGBoostBenchmark._get_fidelity_choices(n_estimators_choice='variable', subsample_choice='fixed') ) return fidelity_space + +class XGBoostBenchmarkMO(_XGBoostBenchmarkBase, MO_MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(XGBoostBenchmarkMO, self).__init__(task_id, rng, valid_size, data_path) + -__all__ = ['XGBoostBenchmarkBB', 'XGBoostBenchmarkMF', 'XGBoostBenchmark'] +__all__ = ['XGBoostBenchmarkBB', 'XGBoostBenchmarkMF', 'XGBoostBenchmark', 'XGBoostBenchmarkMO'] \ No newline at end of file diff --git a/hpobench/container/benchmarks/ml/histgb_benchmark.py b/hpobench/container/benchmarks/ml/histgb_benchmark.py index dc7af088..84b6f459 100644 --- a/hpobench/container/benchmarks/ml/histgb_benchmark.py +++ b/hpobench/container/benchmarks/ml/histgb_benchmark.py @@ -13,6 +13,12 @@ def __init__(self, **kwargs): kwargs['latest'] = kwargs.get('container_tag', '0.0.1') super(HistGBBenchmark, self).__init__(**kwargs) +class HistGBBenchmarkMO(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'HistGBBenchmarkMO') + kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') + kwargs['latest'] = kwargs.get('container_tag', '0.0.2') + super(HistGBBenchmark, self).__init__(**kwargs) class HistGBBenchmarkBB(AbstractBenchmarkClient): def __init__(self, **kwargs): @@ -30,4 +36,4 @@ def __init__(self, **kwargs): super(HistGBBenchmarkMF, self).__init__(**kwargs) -__all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF'] +__all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF', 'HistGBBenchmarkMO'] \ No newline at end of file diff --git a/hpobench/container/benchmarks/ml/lr_benchmark.py b/hpobench/container/benchmarks/ml/lr_benchmark.py index 979cda3e..e07b1f79 100644 --- a/hpobench/container/benchmarks/ml/lr_benchmark.py +++ b/hpobench/container/benchmarks/ml/lr_benchmark.py @@ -13,6 +13,13 @@ def __init__(self, **kwargs): kwargs['latest'] = kwargs.get('container_tag', '0.0.1') super(LRBenchmark, self).__init__(**kwargs) +class LRBenchmarkMO(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'LRBenchmarkMO') + kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') + kwargs['latest'] = kwargs.get('container_tag', '0.0.2') + super(LRBenchmarkMO, self).__init__(**kwargs) + class LRBenchmarkBB(AbstractBenchmarkClient): def __init__(self, **kwargs): @@ -30,4 +37,4 @@ def __init__(self, **kwargs): super(LRBenchmarkMF, self).__init__(**kwargs) -__all__ = ['LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF'] +__all__ = ['LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF', 'LRBenchmarkMO'] \ No newline at end of file diff --git a/hpobench/container/benchmarks/ml/nn_benchmark.py b/hpobench/container/benchmarks/ml/nn_benchmark.py index 04955e82..8b96f946 100644 --- a/hpobench/container/benchmarks/ml/nn_benchmark.py +++ b/hpobench/container/benchmarks/ml/nn_benchmark.py @@ -13,6 +13,12 @@ def __init__(self, **kwargs): kwargs['latest'] = kwargs.get('container_tag', '0.0.1') super(NNBenchmark, self).__init__(**kwargs) +class NNBenchmarkMO(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'NNBenchmarkMO') + kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') + kwargs['latest'] = kwargs.get('container_tag', '0.0.2') + super(NNBenchmarkMO, self).__init__(**kwargs) class NNBenchmarkBB(AbstractBenchmarkClient): def __init__(self, **kwargs): @@ -30,4 +36,4 @@ def __init__(self, **kwargs): super(NNBenchmarkMF, self).__init__(**kwargs) -__all__ = ['NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF'] +__all__ = ['NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF', 'NNBenchmarkMO'] \ No newline at end of file diff --git a/hpobench/container/benchmarks/ml/rf_benchmark.py b/hpobench/container/benchmarks/ml/rf_benchmark.py index a414349d..a3ca46ba 100644 --- a/hpobench/container/benchmarks/ml/rf_benchmark.py +++ b/hpobench/container/benchmarks/ml/rf_benchmark.py @@ -14,6 +14,15 @@ def __init__(self, **kwargs): super(RandomForestBenchmark, self).__init__(**kwargs) + +class RandomForestBenchmarkMO(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'RandomForestBenchmarkMO') + kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') + kwargs['latest'] = kwargs.get('container_tag', '0.0.2') + super(RandomForestBenchmarkMO, self).__init__(**kwargs) + + class RandomForestBenchmarkBB(AbstractBenchmarkClient): def __init__(self, **kwargs): kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'RandomForestBenchmarkBB') @@ -30,4 +39,4 @@ def __init__(self, **kwargs): super(RandomForestBenchmarkMF, self).__init__(**kwargs) -__all__ = ['RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF'] +__all__ = ['RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF', 'RandomForestBenchmarkMO'] \ No newline at end of file diff --git a/hpobench/container/benchmarks/ml/svm_benchmark.py b/hpobench/container/benchmarks/ml/svm_benchmark.py index 7547a81a..46c951a8 100644 --- a/hpobench/container/benchmarks/ml/svm_benchmark.py +++ b/hpobench/container/benchmarks/ml/svm_benchmark.py @@ -13,6 +13,13 @@ def __init__(self, **kwargs): kwargs['latest'] = kwargs.get('container_tag', '0.0.1') super(SVMBenchmark, self).__init__(**kwargs) +class SVMBenchmarkMO(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'SVMBenchmarkMO') + kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') + kwargs['latest'] = kwargs.get('container_tag', '0.0.2') + super(SVMBenchmarkMO, self).__init__(**kwargs) + class SVMBenchmarkMF(AbstractBenchmarkClient): def __init__(self, **kwargs): @@ -30,4 +37,4 @@ def __init__(self, **kwargs): super(SVMBenchmarkBB, self).__init__(**kwargs) -__all__ = ['SVMBenchmark', 'SVMBenchmarkMF', 'SVMBenchmarkBB'] +__all__ = ['SVMBenchmark', 'SVMBenchmarkMF', 'SVMBenchmarkBB', 'SVMBenchmarkMO'] \ No newline at end of file diff --git a/hpobench/container/benchmarks/ml/xgboost_benchmark.py b/hpobench/container/benchmarks/ml/xgboost_benchmark.py index c82ea606..af1e99cc 100644 --- a/hpobench/container/benchmarks/ml/xgboost_benchmark.py +++ b/hpobench/container/benchmarks/ml/xgboost_benchmark.py @@ -13,6 +13,15 @@ def __init__(self, **kwargs): kwargs['latest'] = kwargs.get('container_tag', '0.0.1') super(XGBoostBenchmark, self).__init__(**kwargs) +class XGBoostBenchmarkMO(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'XGBoostBenchmarkMO') + kwargs['container_name'] = kwargs.get('container_name', 'mo_ml_mmfb') + kwargs['container_source'] = 'oras://gitlab.tf.uni-freiburg.de:5050/sharmaa/hpobench-registry' + kwargs['container_tag'] = '0.0.2' + kwargs['latest'] = '0.0.2' #kwargs.get('container_tag', '0.0.2') + super(XGBoostBenchmarkMO, self).__init__(**kwargs) + class XGBoostBenchmarkBB(AbstractBenchmarkClient): def __init__(self, **kwargs): @@ -38,4 +47,4 @@ def __init__(self, **kwargs): super(XGBoostSearchSpace3Benchmark, self).__init__(**kwargs) -__all__ = ['XGBoostBenchmark', 'XGBoostBenchmarkBB', 'XGBoostBenchmarkMF'] +__all__ = ['XGBoostBenchmark', 'XGBoostBenchmarkBB', 'XGBoostBenchmarkMF', 'XGBoostBenchmarkMO'] \ No newline at end of file diff --git a/hpobench/container/client_abstract_benchmark.py b/hpobench/container/client_abstract_benchmark.py index d2963c00..f6091635 100644 --- a/hpobench/container/client_abstract_benchmark.py +++ b/hpobench/container/client_abstract_benchmark.py @@ -222,7 +222,8 @@ def download_container(container_dir, container_name, container_source, containe # Currently, we can't see the available container tags on gitlab. Therefore, we create for each # "tag" a new entry in the registry. This might change in the future. But as long as we don't have # a fix for this, we need to map the container tag differently. - if container_source.startswith('oras://gitlab.tf.uni-freiburg.de:5050/muelleph/hpobench-registry'): + if (container_source.startswith('oras://gitlab.tf.uni-freiburg.de:5050/muelleph/hpobench-registry')) or \ + (container_source.startswith('oras://gitlab.tf.uni-freiburg.de:5050/sharmaa/hpobench-registry')): cmd += f'{container_source}/{container_name.lower()}/{container_tag}:latest' else: cmd += f'{container_source}/{container_name.lower()}:{container_tag}' diff --git a/hpobench/dependencies/ml/ml_benchmark_template.py b/hpobench/dependencies/ml/ml_benchmark_template.py index 7cef515f..4bf33e77 100644 --- a/hpobench/dependencies/ml/ml_benchmark_template.py +++ b/hpobench/dependencies/ml/ml_benchmark_template.py @@ -11,6 +11,8 @@ from hpobench.abstract_benchmark import AbstractBenchmark from hpobench.dependencies.ml.data_manager import OpenMLDataManager from hpobench.util.rng_helper import get_rng +from hpobench.abstract_benchmark import AbstractMultiObjectiveBenchmark, AbstractSingleObjectiveBenchmark +from typing import Union, Dict, List, Text, Tuple metrics = dict( acc=accuracy_score, @@ -27,7 +29,7 @@ ) -class MLBenchmark(AbstractBenchmark): +class _MLBenchmarkBase(): _issue_tasks = [3917, 3945] def __init__( @@ -38,7 +40,7 @@ def __init__( data_path: Union[str, Path, None] = None, global_seed: int = 1 ): - super(MLBenchmark, self).__init__(rng=rng) + super(_MLBenchmarkBase, self).__init__(rng=rng) if isinstance(rng, int): self.seed = rng @@ -197,16 +199,38 @@ def _train_objective(self, score_cost[k] = time.time() - _start train_loss = 1 - scores["acc"] return model, model_fit_time, train_loss, scores, score_cost - - # pylint: disable=arguments-differ - @AbstractBenchmark.check_parameters - def objective_function(self, - configuration: Union[CS.Configuration, Dict], - fidelity: Union[CS.Configuration, Dict, None] = None, - shuffle: bool = False, - rng: Union[np.random.RandomState, int, None] = None, - **kwargs) -> Dict: + + @AbstractMultiObjectiveBenchmark.check_parameters + def _mo_objective_function(self, + configuration: Union[CS.Configuration, Dict], + fidelity: Union[CS.Configuration, Dict, None] = None, + shuffle: bool = False, + rng: Union[np.random.RandomState, int, None] = None, + **kwargs) -> Dict: """Function that evaluates a 'config' on a 'fidelity' on the validation set + + Parameters + ---------- + configuration : Dict, CS.Configuration + fidelity: Dict, None + Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None. + rng : np.random.RandomState, int, None + Random seed to use in the benchmark. + + To prevent overfitting on a single seed, it is possible to pass a + parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. + If this parameter is not given, the default random state is used. + kwargs + + Returns + ------- + Dict - + function_value : + ToDo: add return values + cost : runtime + info : Dict + fidelity : used fidelities in this evaluation + """ model, model_fit_time, train_loss, train_scores, train_score_cost = self._train_objective( configuration, fidelity, shuffle, rng, evaluation="val" @@ -218,6 +242,8 @@ def objective_function(self, val_scores[k] = v(model, self.valid_X, self.valid_y) val_score_cost[k] = time.time() - _start val_loss = 1 - val_scores["acc"] + val_sensitivity = (val_scores['f1'] * val_scores['precision']) / (2 * val_scores['precision'] - val_scores['f1'] ) + val_scores['fnr'] = 1 - val_sensitivity test_scores = dict() test_score_cost = dict() @@ -227,6 +253,9 @@ def objective_function(self, test_score_cost[k] = time.time() - _start test_loss = 1 - test_scores["acc"] + test_sensitivity = (test_scores['f1'] * test_scores['precision']) / (2 * test_scores['precision'] - test_scores['f1'] ) + test_scores['fnr'] = 1 - test_sensitivity + info = { 'train_loss': train_loss, 'val_loss': val_loss, @@ -243,15 +272,22 @@ def objective_function(self, 'config': configuration, } - return { - 'function_value': info['val_loss'], + result_dict = { + 'function_value': { + # The original benchmark returned the accuracy with range [0, 100]. + # We cast it to a minimization problem with range [0-1] to have a more standardized return value. + 'val_loss': val_loss, + 'fnr': val_scores['fnr'], + + }, 'cost': model_fit_time + info['val_costs']['acc'], 'info': info } - # pylint: disable=arguments-differ - @AbstractBenchmark.check_parameters - def objective_function_test(self, + return result_dict + + @AbstractMultiObjectiveBenchmark.check_parameters + def _mo_objective_function_test(self, configuration: Union[CS.Configuration, Dict], fidelity: Union[CS.Configuration, Dict, None] = None, shuffle: bool = False, @@ -270,6 +306,9 @@ def objective_function_test(self, test_score_cost[k] = time.time() - _start test_loss = 1 - test_scores["acc"] + test_sensitivity = (test_scores['f1'] * test_scores['precision']) / (2 * test_scores['precision'] - test_scores['f1'] ) + test_scores['fnr'] = 1 - test_sensitivity + info = { 'train_loss': train_loss, 'val_loss': None, @@ -287,7 +326,99 @@ def objective_function_test(self, } return { - 'function_value': float(info['test_loss']), + 'function_value': { + # The original benchmark returned the accuracy with range [0, 100]. + # We cast it to a minimization problem with range [0-1] to have a more standardized return value. + 'test_loss': float(info['test_loss']), + # 'f1': test_scores['f1'], + # 'precision': test_scores['precision'], + # 'balanced_accuracy': test_scores['bal_acc'], + 'fnr': test_scores['fnr'], + }, 'cost': float(model_fit_time + info['test_costs']['acc']), 'info': info } + + +class MLBenchmark(_MLBenchmarkBase, AbstractSingleObjectiveBenchmark): + + + @AbstractSingleObjectiveBenchmark.check_parameters + def objective_function(self, + configuration: Union[CS.Configuration, Dict], + fidelity: Union[CS.Configuration, Dict, None] = None, + shuffle: bool = False, + rng: Union[np.random.RandomState, int, None] = None, + **kwargs) -> Dict: + """Function that evaluates a 'config' on a 'fidelity' on the validation set + + Parameters + ---------- + configuration : Dict, CS.Configuration + fidelity: Dict, None + Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None. + rng : np.random.RandomState, int, None + Random seed to use in the benchmark. + + To prevent overfitting on a single seed, it is possible to pass a + parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. + If this parameter is not given, the default random state is used. + kwargs + + Returns + ------- + Dict - + function_value : + ToDo: add return values + cost : runtime + info : Dict + fidelity : used fidelities in this evaluation + + """ + results = self._mo_objective_function( + configuration=configuration, fidelity=fidelity, rng=rng, **kwargs + ) + results['function_value'] = results['function_value']['val_loss'] + return results + + @AbstractSingleObjectiveBenchmark.check_parameters + def objective_function_test(self, + configuration: Union[CS.Configuration, Dict], + fidelity: Union[CS.Configuration, Dict, None] = None, + shuffle: bool = False, + rng: Union[np.random.RandomState, int, None] = None, + **kwargs) -> Dict: + """Function that evaluates a 'config' on a 'fidelity' on the test set + """ + + results = self._mo_objective_function_test( + configuration=configuration, fidelity=fidelity, rng=rng, **kwargs + ) + results['function_value'] = results['function_value']['test_loss'] + return results + +class MO_MLBenchmark(_MLBenchmarkBase, AbstractMultiObjectiveBenchmark): + + @AbstractMultiObjectiveBenchmark.check_parameters + def objective_function(self, + configuration: Union[CS.Configuration, Dict], + fidelity: Union[CS.Configuration, Dict, None] = None, + shuffle: bool = False, + rng: Union[np.random.RandomState, int, None] = None, + **kwargs) -> Dict: + return self._mo_objective_function(configuration=configuration, fidelity=fidelity,shuffle=shuffle, rng=rng, + **kwargs) + + @AbstractMultiObjectiveBenchmark.check_parameters + def objective_function_test(self, + configuration: Union[CS.Configuration, Dict], + fidelity: Union[CS.Configuration, Dict, None] = None, + shuffle: bool = False, + rng: Union[np.random.RandomState, int, None] = None, + **kwargs) -> Dict: + return self._mo_objective_function_test(configuration=configuration, fidelity=fidelity, shuffle=shuffle, rng=rng, + **kwargs) + @staticmethod + def get_objective_names() -> List[str]: + return ['val_loss', 'fnr', bal_acc', 'f1', 'precision'] + From 59613d03d1317aff0fa0b2b313d9a26be234e765 Mon Sep 17 00:00:00 2001 From: Ayushi Date: Mon, 17 Apr 2023 11:26:15 +0200 Subject: [PATCH 04/17] minor fixes --- hpobench/benchmarks/ml/lr_benchmark.py | 2 +- hpobench/benchmarks/ml/nn_benchmark.py | 2 +- hpobench/dependencies/ml/ml_benchmark_template.py | 13 ++++++------- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/hpobench/benchmarks/ml/lr_benchmark.py b/hpobench/benchmarks/ml/lr_benchmark.py index b50e8f35..63d4169e 100755 --- a/hpobench/benchmarks/ml/lr_benchmark.py +++ b/hpobench/benchmarks/ml/lr_benchmark.py @@ -147,7 +147,7 @@ def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationS ) return fidelity_space -class LRBenchmarkMO(_LRBenchmarkBase, MO_MLBenchmark): +class LRBenchmarkMO(LRBenchmark, MO_MLBenchmark): def __init__(self, task_id: int, rng: Union[np.random.RandomState, int, None] = None, diff --git a/hpobench/benchmarks/ml/nn_benchmark.py b/hpobench/benchmarks/ml/nn_benchmark.py index d22d2498..765da72b 100755 --- a/hpobench/benchmarks/ml/nn_benchmark.py +++ b/hpobench/benchmarks/ml/nn_benchmark.py @@ -146,7 +146,7 @@ def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationS ) return fidelity_space -class NNBenchmarkMO(_NNBenchmarkBase, MO_MLBenchmark): +class NNBenchmarkMO(NNBenchmark, MO_MLBenchmark): def __init__(self, task_id: int, rng: Union[np.random.RandomState, int, None] = None, diff --git a/hpobench/dependencies/ml/ml_benchmark_template.py b/hpobench/dependencies/ml/ml_benchmark_template.py index 9ed139ba..81ef5be2 100755 --- a/hpobench/dependencies/ml/ml_benchmark_template.py +++ b/hpobench/dependencies/ml/ml_benchmark_template.py @@ -243,7 +243,7 @@ def _mo_objective_function(self, val_score_cost[k] = time.time() - _start val_loss = 1 - val_scores["acc"] val_sensitivity = (val_scores['f1'] * val_scores['precision']) / (2 * val_scores['precision'] - val_scores['f1'] ) - val_scores['false_negative_rate'] = 1 - val_sensitivity + val_scores['fnr'] = 1 - val_sensitivity test_scores = dict() test_score_cost = dict() @@ -254,7 +254,7 @@ def _mo_objective_function(self, test_loss = 1 - test_scores["acc"] test_sensitivity = (test_scores['f1'] * test_scores['precision']) / (2 * test_scores['precision'] - test_scores['f1'] ) - test_scores['false_negative_rate'] = 1 - test_sensitivity + test_scores['fnr'] = 1 - test_sensitivity info = { 'train_loss': train_loss, @@ -277,7 +277,7 @@ def _mo_objective_function(self, # The original benchmark returned the accuracy with range [0, 100]. # We cast it to a minimization problem with range [0-1] to have a more standardized return value. 'val_loss': val_loss, - 'sensitivity': val_scores['false_negative_rate'], + 'fnr': val_scores['fnr'], }, 'cost': model_fit_time + info['val_costs']['acc'], @@ -307,7 +307,7 @@ def _mo_objective_function_test(self, test_loss = 1 - test_scores["acc"] test_sensitivity = (test_scores['f1'] * test_scores['precision']) / (2 * test_scores['precision'] - test_scores['f1'] ) - test_scores['false_negative_rate'] = 1 - test_sensitivity + test_scores['fnr'] = 1 - test_sensitivity info = { 'train_loss': train_loss, @@ -333,7 +333,7 @@ def _mo_objective_function_test(self, # 'f1': test_scores['f1'], # 'precision': test_scores['precision'], # 'balanced_accuracy': test_scores['bal_acc'], - 'sensitivity': test_scores['false_negative_rate'], + 'fnr': test_scores['fnr'], }, 'cost': float(model_fit_time + info['test_costs']['acc']), 'info': info @@ -420,6 +420,5 @@ def objective_function_test(self, **kwargs) @staticmethod def get_objective_names() -> List[str]: - return ['val_loss', 'bal_acc', 'f1', 'precision', 'sensitivity'] - + return ['val_loss', 'fnr'] From dabe38ff74cc439455ddb4bed37ac902dfc98f8f Mon Sep 17 00:00:00 2001 From: Ayushi Date: Mon, 17 Apr 2023 12:32:21 +0200 Subject: [PATCH 05/17] fix class names --- hpobench/benchmarks/ml/lr_benchmark.py | 4 ++-- hpobench/benchmarks/ml/nn_benchmark.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hpobench/benchmarks/ml/lr_benchmark.py b/hpobench/benchmarks/ml/lr_benchmark.py index 63d4169e..369885b1 100755 --- a/hpobench/benchmarks/ml/lr_benchmark.py +++ b/hpobench/benchmarks/ml/lr_benchmark.py @@ -147,14 +147,14 @@ def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationS ) return fidelity_space -class LRBenchmarkMO(LRBenchmark, MO_MLBenchmark): +class LRBenchmarkMO(_LRBenchmarkBase, MO_MLBenchmark): def __init__(self, task_id: int, rng: Union[np.random.RandomState, int, None] = None, valid_size: float = 0.33, data_path: Union[str, None] = None): - super(LRBenchmark, self).__init__(task_id, rng, valid_size, data_path) + super(LRBenchmarkMO, self).__init__(task_id, rng, valid_size, data_path) self.cache_size = 500 diff --git a/hpobench/benchmarks/ml/nn_benchmark.py b/hpobench/benchmarks/ml/nn_benchmark.py index 765da72b..cae60e12 100755 --- a/hpobench/benchmarks/ml/nn_benchmark.py +++ b/hpobench/benchmarks/ml/nn_benchmark.py @@ -146,12 +146,12 @@ def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationS ) return fidelity_space -class NNBenchmarkMO(NNBenchmark, MO_MLBenchmark): +class NNBenchmarkMO(_NNBenchmarkBase, MO_MLBenchmark): def __init__(self, task_id: int, rng: Union[np.random.RandomState, int, None] = None, valid_size: float = 0.33, data_path: Union[str, None] = None): - super(NNBenchmark, self).__init__(task_id, rng, valid_size, data_path) + super(NNBenchmarkMO, self).__init__(task_id, rng, valid_size, data_path) __all__ = ['NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF', 'NNBenchmarkMO'] From 62f9461334f100efa2bb29ca41c224bfdf0e0032 Mon Sep 17 00:00:00 2001 From: Ayushi Date: Tue, 25 Apr 2023 13:46:56 +0200 Subject: [PATCH 06/17] add precision in objective list --- hpobench/benchmarks/ml/lr_benchmark.py | 1 - .../dependencies/ml/ml_benchmark_template.py | 20 +++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/hpobench/benchmarks/ml/lr_benchmark.py b/hpobench/benchmarks/ml/lr_benchmark.py index 369885b1..8357bd66 100755 --- a/hpobench/benchmarks/ml/lr_benchmark.py +++ b/hpobench/benchmarks/ml/lr_benchmark.py @@ -153,7 +153,6 @@ def __init__(self, rng: Union[np.random.RandomState, int, None] = None, valid_size: float = 0.33, data_path: Union[str, None] = None): - super(LRBenchmarkMO, self).__init__(task_id, rng, valid_size, data_path) self.cache_size = 500 diff --git a/hpobench/dependencies/ml/ml_benchmark_template.py b/hpobench/dependencies/ml/ml_benchmark_template.py index 81ef5be2..e963855a 100755 --- a/hpobench/dependencies/ml/ml_benchmark_template.py +++ b/hpobench/dependencies/ml/ml_benchmark_template.py @@ -1,14 +1,11 @@ import time from pathlib import Path from typing import Union, Dict, Iterable - import ConfigSpace as CS import numpy as np import pandas as pd from sklearn.metrics import make_scorer, accuracy_score, balanced_accuracy_score, \ precision_score, f1_score - -from hpobench.abstract_benchmark import AbstractBenchmark from hpobench.dependencies.ml.data_manager import OpenMLDataManager from hpobench.util.rng_helper import get_rng from hpobench.abstract_benchmark import AbstractMultiObjectiveBenchmark, AbstractSingleObjectiveBenchmark @@ -244,6 +241,7 @@ def _mo_objective_function(self, val_loss = 1 - val_scores["acc"] val_sensitivity = (val_scores['f1'] * val_scores['precision']) / (2 * val_scores['precision'] - val_scores['f1'] ) val_scores['fnr'] = 1 - val_sensitivity + val_scores['fpr'] = 1 - val_scores['precision'] test_scores = dict() test_score_cost = dict() @@ -255,6 +253,7 @@ def _mo_objective_function(self, test_sensitivity = (test_scores['f1'] * test_scores['precision']) / (2 * test_scores['precision'] - test_scores['f1'] ) test_scores['fnr'] = 1 - test_sensitivity + test_scores['fpr'] = 1 - test_scores['precision'] info = { 'train_loss': train_loss, @@ -276,8 +275,9 @@ def _mo_objective_function(self, 'function_value': { # The original benchmark returned the accuracy with range [0, 100]. # We cast it to a minimization problem with range [0-1] to have a more standardized return value. - 'val_loss': val_loss, - 'fnr': val_scores['fnr'], + 'val_loss': float(val_loss), + 'fnr': float(val_scores['fnr']), + 'fpr': float(val_scores['fpr']), }, 'cost': model_fit_time + info['val_costs']['acc'], @@ -308,7 +308,7 @@ def _mo_objective_function_test(self, test_sensitivity = (test_scores['f1'] * test_scores['precision']) / (2 * test_scores['precision'] - test_scores['f1'] ) test_scores['fnr'] = 1 - test_sensitivity - + test_scores['fpr'] = 1 - test_scores['precision'] info = { 'train_loss': train_loss, 'val_loss': None, @@ -331,9 +331,9 @@ def _mo_objective_function_test(self, # We cast it to a minimization problem with range [0-1] to have a more standardized return value. 'test_loss': float(info['test_loss']), # 'f1': test_scores['f1'], - # 'precision': test_scores['precision'], + 'fpr': float(test_scores['fpr']), # 'balanced_accuracy': test_scores['bal_acc'], - 'fnr': test_scores['fnr'], + 'fnr': float(test_scores['fnr']), }, 'cost': float(model_fit_time + info['test_costs']['acc']), 'info': info @@ -406,7 +406,7 @@ def objective_function(self, shuffle: bool = False, rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: - return self._mo_objective_function(configuration=configuration, fidelity=fidelity,shuffle=shuffle, rng=rng, + return self._mo_objective_function(configuration=configuration, fidelity=fidelity, shuffle=shuffle, rng=rng, **kwargs) @AbstractMultiObjectiveBenchmark.check_parameters @@ -420,5 +420,5 @@ def objective_function_test(self, **kwargs) @staticmethod def get_objective_names() -> List[str]: - return ['val_loss', 'fnr'] + return ['val_loss', 'fnr', 'fpr'] From dd6cfb5d25fb3fb0986d6c9e1ccea98dd0a01cd5 Mon Sep 17 00:00:00 2001 From: ayushi-3536 Date: Tue, 9 May 2023 23:49:42 +0200 Subject: [PATCH 07/17] added mo setting for tabular --- .../recipes/ml/Singularity.mo_ml_mmfb | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 hpobench/container/recipes/ml/Singularity.mo_ml_mmfb diff --git a/hpobench/container/recipes/ml/Singularity.mo_ml_mmfb b/hpobench/container/recipes/ml/Singularity.mo_ml_mmfb new file mode 100644 index 00000000..0249e727 --- /dev/null +++ b/hpobench/container/recipes/ml/Singularity.mo_ml_mmfb @@ -0,0 +1,25 @@ +Bootstrap: docker +From: python:3.8-slim + +%labels +MAINTAINER sharmaa@informatik.uni-freiburg.de +VERSION v0.0.2 + +%post + apt update -y \ + && apt install build-essential git -y \ + + cd /home \ + && git clone https://github.com/ayushi-3536/HPOBench.git \ + && cd HPOBench \ + && git checkout mo_tabular \ + && pip install ".[ml_mfbb]" \ + && cd / \ + && mkdir /var/lib/hpobench/ \ + && chmod -R 777 /var/lib/hpobench/ \ + && pip cache purge \ + && rm -rf /var/lib/apt/lists/* + + +%runscript + python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py ml $@ From 261563c76a9dc6cac41166a6fc156a37343e2c33 Mon Sep 17 00:00:00 2001 From: ayushi-3536 Date: Wed, 10 May 2023 00:34:03 +0200 Subject: [PATCH 08/17] added tabular mo setup --- hpobench/benchmarks/ml/__init__.py | 8 +- hpobench/benchmarks/ml/lr_benchmark.py | 39 +++- hpobench/benchmarks/ml/nn_benchmark.py | 2 +- hpobench/benchmarks/ml/tabular_benchmark.py | 207 +++++++++++++----- .../dependencies/ml/ml_benchmark_template.py | 21 +- 5 files changed, 207 insertions(+), 70 deletions(-) diff --git a/hpobench/benchmarks/ml/__init__.py b/hpobench/benchmarks/ml/__init__.py index fcdac8d6..5102cd4c 100644 --- a/hpobench/benchmarks/ml/__init__.py +++ b/hpobench/benchmarks/ml/__init__.py @@ -1,10 +1,10 @@ -from hpobench.benchmarks.ml.histgb_benchmark import HistGBBenchmark, HistGBBenchmarkBB, HistGBBenchmarkMF, HistGBBenchmarkMO +from hpobench.benchmarks.ml.histgb_benchmark import HistGBBenchmark, HistGBBenchmarkBB, HistGBBenchmarkMF from hpobench.benchmarks.ml.lr_benchmark import LRBenchmark, LRBenchmarkBB, LRBenchmarkMF, LRBenchmarkMO from hpobench.benchmarks.ml.nn_benchmark import NNBenchmark, NNBenchmarkBB, NNBenchmarkMF, NNBenchmarkMO from hpobench.benchmarks.ml.rf_benchmark import RandomForestBenchmark, RandomForestBenchmarkBB, \ RandomForestBenchmarkMF, RandomForestBenchmarkMO from hpobench.benchmarks.ml.svm_benchmark import SVMBenchmark, SVMBenchmarkBB, SVMBenchmarkMF, SVMBenchmarkMO -from hpobench.benchmarks.ml.tabular_benchmark import TabularBenchmark +from hpobench.benchmarks.ml.tabular_benchmark import TabularBenchmark, TabularBenchmarkMO try: from hpobench.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark, XGBoostBenchmarkBB, XGBoostBenchmarkMF, XGBoostBenchmarkMO @@ -12,11 +12,11 @@ pass -__all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF', 'HistGBBenchmarkMO', +__all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF', 'LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF', 'LRBenchmarkMO', 'NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF', 'NNBenchmarkMO', 'RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF', 'RandomForestBenchmarkMO', 'SVMBenchmark', 'SVMBenchmarkBB', 'SVMBenchmarkMF', 'SVMBenchmarkMO', - 'TabularBenchmark', + 'TabularBenchmark', 'TabularBenchmarkMO', 'XGBoostBenchmark', 'XGBoostBenchmarkBB', 'XGBoostBenchmarkMF', 'XGBoostBenchmarkMO', ] \ No newline at end of file diff --git a/hpobench/benchmarks/ml/lr_benchmark.py b/hpobench/benchmarks/ml/lr_benchmark.py index 8c317111..6019a29f 100644 --- a/hpobench/benchmarks/ml/lr_benchmark.py +++ b/hpobench/benchmarks/ml/lr_benchmark.py @@ -14,19 +14,28 @@ from ConfigSpace.hyperparameters import Hyperparameter from sklearn.linear_model import SGDClassifier -from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark +from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark, MO_MLBenchmark +''' +Changelog: +========== +0.0.2: +* Add the multiobjective version of this benchmark by returning val loss, precision, f1 and balanced accuracy -__version__ = '0.0.1' +0.0.1: +* First implementation +''' +__version__ = '0.0.2' -class LRBenchmark(MLBenchmark): + +class _LRBenchmarkBase: def __init__(self, task_id: int, rng: Union[np.random.RandomState, int, None] = None, valid_size: float = 0.33, data_path: Union[str, None] = None): - super(LRBenchmark, self).__init__(task_id, rng, valid_size, data_path) + super(_LRBenchmarkBase, self).__init__(task_id, rng, valid_size, data_path) self.cache_size = 500 @staticmethod @@ -107,6 +116,17 @@ def init_model(self, config: Union[CS.Configuration, Dict], ) return model +class LRBenchmark(_LRBenchmarkBase, MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + + super(LRBenchmark, self).__init__(task_id, rng, valid_size, data_path) + self.cache_size = 500 + + class LRBenchmarkBB(LRBenchmark): def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -126,6 +146,15 @@ def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationS LRBenchmark._get_fidelity_choices(iter_choice='variable', subsample_choice='fixed') ) return fidelity_space + +class LRBenchmarkMO(_LRBenchmarkBase, MO_MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(LRBenchmarkMO, self).__init__(task_id, rng, valid_size, data_path) + self.cache_size = 500 -__all__ = ['LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF'] +__all__ = ['LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF', 'LRBenchmarkMO'] \ No newline at end of file diff --git a/hpobench/benchmarks/ml/nn_benchmark.py b/hpobench/benchmarks/ml/nn_benchmark.py index a16d967b..4febcccc 100644 --- a/hpobench/benchmarks/ml/nn_benchmark.py +++ b/hpobench/benchmarks/ml/nn_benchmark.py @@ -152,6 +152,6 @@ def __init__(self, rng: Union[np.random.RandomState, int, None] = None, valid_size: float = 0.33, data_path: Union[str, None] = None): - super(NNBenchmark, self).__init__(task_id, rng, valid_size, data_path) + super(NNBenchmarkMO, self).__init__(task_id, rng, valid_size, data_path) __all__ = ['NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF', 'NNBenchmarkMO'] \ No newline at end of file diff --git a/hpobench/benchmarks/ml/tabular_benchmark.py b/hpobench/benchmarks/ml/tabular_benchmark.py index 72e5fb31..c7585ad7 100644 --- a/hpobench/benchmarks/ml/tabular_benchmark.py +++ b/hpobench/benchmarks/ml/tabular_benchmark.py @@ -13,14 +13,21 @@ import numpy as np from ConfigSpace.read_and_write import json as json_cs -from hpobench.abstract_benchmark import AbstractBenchmark +from hpobench.abstract_benchmark import AbstractSingleObjectiveBenchmark, AbstractMultiObjectiveBenchmark, AbstractBenchmark from hpobench.dependencies.ml.ml_benchmark_template import metrics from hpobench.util.data_manager import TabularDataManager -__version__ = '0.0.1' +__version__ = '0.0.2' +""" +Changelog: +========== +0.0.2: Added MO Tabular Benchmark + +0.0.1: First implementation of the Tabular Benchmark. +""" -class TabularBenchmark(AbstractBenchmark): +class _TabularBenchmarkBase: def __init__(self, model: str, task_id: int, @@ -42,33 +49,8 @@ def __init__(self, self.original_cs = json_cs.read(self.config_spaces['x']) self.original_fs = json_cs.read(self.config_spaces['z']) - super(TabularBenchmark, self).__init__(rng=rng, **kwargs) - - # pylint: disable=arguments-differ - @AbstractBenchmark.check_parameters - def objective_function(self, - configuration: Union[CS.Configuration, Dict], - fidelity: Union[Dict, CS.Configuration, None] = None, - rng: Union[np.random.RandomState, int, None] = None, - seed: Union[int, None] = None, - metric: Union[str, None] = 'acc', - **kwargs) -> Dict: - - result = self._objective(configuration, fidelity, seed, metric, evaluation="val") - return result - - # pylint: disable=arguments-differ - @AbstractBenchmark.check_parameters - def objective_function_test(self, - configuration: Union[CS.Configuration, Dict], - fidelity: Union[Dict, CS.Configuration, None] = None, - rng: Union[np.random.RandomState, int, None] = None, - seed: Union[int, None] = None, - metric: Union[str, None] = 'acc', - **kwargs) -> Dict: + super(_TabularBenchmarkBase, self).__init__(rng=rng) - result = self._objective(configuration, fidelity, seed, metric, evaluation="test") - return result # pylint: disable=arguments-differ def get_configuration_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -142,26 +124,36 @@ def get_fidelity_range(self) -> List: def _search_dataframe(self, row_dict, df): # https://stackoverflow.com/a/46165056/8363967 mask = np.array([True] * df.shape[0]) - for i, param in enumerate(df.drop("result", axis=1).columns): + for i, param in enumerate(df.drop(df.filter(regex='result.').columns, axis=1)): mask *= df[param].values == row_dict[param] idx = np.where(mask) assert len(idx) == 1, 'The query has resulted into mulitple matches. This should not happen. ' \ f'The Query was {row_dict}' idx = idx[0][0] - result = df.iloc[idx]["result"] - return result - - def _objective( + result = df.iloc[idx] + result = result.filter(regex='result.') + #convert the result to dict + resultdict = result.to_dict() + result={} + for key, value in resultdict.items(): + keys = key.split('.') + d = result + for k in keys[:-1]: + if k not in d: + d[k] = {} + d = d[k] + d[keys[-1]] = value + + return result['result'] + + def _moobjective( self, config: Dict, fidelity: Dict, seed: Union[int, None] = None, - metric: Union[str, None] = "acc", evaluation: Union[str, None] = "" ) -> Dict: - metric_str = ', '.join(list(metrics.keys())) - assert metric in list(metrics.keys()), f"metric not found among: {metric_str}" score_key = f"{evaluation}_scores" cost_key = f"{evaluation}_scores" @@ -169,7 +161,7 @@ def _objective( for name in self.configuration_space.get_hyperparameter_names(): key_path[str(name)] = config[str(name)] for name in self.fidelity_space.get_hyperparameter_names(): - key_path[str(name)] = fidelity[str(name)] + key_path[str(name)] = float(fidelity[str(name)]) if seed is not None: assert seed in self._seeds_used() @@ -177,19 +169,134 @@ def _objective( else: seeds = self._seeds_used() - loss = [] - costs = 0.0 + + costs = dict(acc=0.0, f1=0.0, precision=0.0, bal_acc=0.0, model_cost=0.0) info = dict() - for seed in seeds: - key_path["seed"] = seed - res = self._search_dataframe(key_path, self.table) - loss.append(1 - res["info"][score_key][metric]) - costs += res["info"]["model_cost"] + res["info"][cost_key][metric] - info[seed] = res["info"] - key_path.pop("seed") - loss = np.mean(loss) - result = dict(function_value=float(loss), cost=costs, info=info) + dict_metrics = dict(acc=[], f1=[], precision=[], bal_acc=[]) + + + + for metric_key, metric_value in metrics.items(): + for seed in seeds: + key_path["seed"] = seed + res = self._search_dataframe(key_path, self.table) + dict_metrics[metric_key].append(res["info"][score_key][metric_key]) + costs["model_cost"] = costs["model_cost"] + res["info"]["model_cost"] + costs[metric_key] = costs[metric_key] + res["info"][cost_key][metric_key] + info[seed] = res["info"] + key_path.pop("seed") + + + result_dict = { + 'function_value': { + # The original benchmark returned the accuracy with range [0, 100]. + # We cast it to a minimization problem with range [0-1] to have a more standardized return value. + 'acc': float(np.mean(dict_metrics['acc'])), + 'precision': float(np.mean(dict_metrics['precision'])), + 'f1': float(np.mean(dict_metrics['f1'])), + 'bal_acc': float(np.mean(dict_metrics['bal_acc'])), + }, + 'cost': { + 'acc': float(costs['acc'] / len(seeds)), + 'precision': float(costs['precision'] / len(seeds)), + 'f1': float(costs['f1'] / len(seeds)), + 'bal_acc': float(costs['bal_acc'] / len(seeds)), + 'model_cost': float(costs['model_cost'] / len(seeds)), + }, + 'info': info + } + return result_dict + + +class TabularBenchmarkMO(_TabularBenchmarkBase, AbstractMultiObjectiveBenchmark): + + @AbstractMultiObjectiveBenchmark.check_parameters + def objective_function(self, configuration: Union[CS.Configuration, Dict], + fidelity: Union[Dict, CS.Configuration, None] = None, + rng: Union[np.random.RandomState, int, None] = None, + seed: Union[int, None] = None, + **kwargs) -> Dict: + result_dict = self._moobjective(config=configuration, fidelity=fidelity, seed=seed, evaluation="val") + sensitivity = (result_dict['function_value']['f1'] * result_dict['function_value']['precision']) / (2 * result_dict['function_value']['precision'] - result_dict['function_value']['f1'] ) + false_negative_rate = 1 - sensitivity + false_positive_rate = 1 - result_dict['function_value']['precision'] + #not considering cost for loss as it is not one of the objectives + cost = result_dict['cost']['model_cost'] + result_dict['cost']['precision'] + result_dict['cost']['f1'] + + result = { + 'function_value': { + 'fnr': float(false_negative_rate), + 'fpr': float(false_positive_rate), + + }, + 'cost': cost, + 'info': result_dict['info'] + } + + return result + + @AbstractMultiObjectiveBenchmark.check_parameters + def objective_function_test(self, + configuration: Union[CS.Configuration, Dict], + fidelity: Union[Dict, CS.Configuration, None] = None, + rng: Union[np.random.RandomState, int, None] = None, + seed: Union[int, None] = None, + **kwargs) -> Dict: + result_dict = self._moobjective(config=configuration, fidelity=fidelity, seed=seed, evaluation="test") + sensitivity = (result_dict['function_value']['f1'] * result_dict['function_value']['precision']) / (2 * result_dict['function_value']['precision'] - result_dict['function_value']['f1'] ) + false_negative_rate = 1 - sensitivity + false_positive_rate = 1 - result_dict['function_value']['precision'] + #not considering cost for loss as it is not one of the objectives + cost = result_dict['cost']['model_cost'] + result_dict['cost']['precision'] + result_dict['cost']['f1'] + + result = { + 'function_value': { + 'fnr': float(false_negative_rate), + 'fpr': float(false_positive_rate), + + }, + 'cost': cost, + 'info': result_dict['info'] + } + + return result + + @staticmethod + def get_objective_names() -> List[str]: + return ['fnr', 'fpr'] + + +class TabularBenchmark(_TabularBenchmarkBase, AbstractSingleObjectiveBenchmark): + # pylint: disable=arguments-differ + @AbstractSingleObjectiveBenchmark.check_parameters + def objective_function(self, + configuration: Union[CS.Configuration, Dict], + fidelity: Union[Dict, CS.Configuration, None] = None, + rng: Union[np.random.RandomState, int, None] = None, + seed: Union[int, None] = None, + metric: Union[str, None] = 'acc', + **kwargs) -> Dict: + + metric_str = ', '.join(list(metrics.keys())) + assert metric in list(metrics.keys()), f"metric not found among: {metric_str}" + result_dict = self._moobjective(configuration, fidelity, seed, evaluation="val") + result = dict(function_value=float(1-result_dict['function_value'][metric]), cost=result_dict['cost'][metric], info=result_dict['info']) return result + # pylint: disable=arguments-differ + @AbstractSingleObjectiveBenchmark.check_parameters + def objective_function_test(self, + configuration: Union[CS.Configuration, Dict], + fidelity: Union[Dict, CS.Configuration, None] = None, + rng: Union[np.random.RandomState, int, None] = None, + seed: Union[int, None] = None, + metric: Union[str, None] = 'acc', + **kwargs) -> Dict: -__all__ = ['TabularBenchmark'] + metric_str = ', '.join(list(metrics.keys())) + assert metric in list(metrics.keys()), f"metric not found among: {metric_str}" + result_dict = self._moobjective(configuration, fidelity, seed, evaluation="test") + result = dict(function_value=float(1-result_dict['function_value'][metric]), cost=result_dict['cost'][metric], info=result_dict['info']) + return result + +__all__ = ['TabularBenchmark', 'TabularBenchmarkMO'] diff --git a/hpobench/dependencies/ml/ml_benchmark_template.py b/hpobench/dependencies/ml/ml_benchmark_template.py index 4bf33e77..56e881da 100644 --- a/hpobench/dependencies/ml/ml_benchmark_template.py +++ b/hpobench/dependencies/ml/ml_benchmark_template.py @@ -1,14 +1,11 @@ import time from pathlib import Path from typing import Union, Dict, Iterable - import ConfigSpace as CS import numpy as np import pandas as pd from sklearn.metrics import make_scorer, accuracy_score, balanced_accuracy_score, \ precision_score, f1_score - -from hpobench.abstract_benchmark import AbstractBenchmark from hpobench.dependencies.ml.data_manager import OpenMLDataManager from hpobench.util.rng_helper import get_rng from hpobench.abstract_benchmark import AbstractMultiObjectiveBenchmark, AbstractSingleObjectiveBenchmark @@ -232,6 +229,7 @@ def _mo_objective_function(self, fidelity : used fidelities in this evaluation """ + model, model_fit_time, train_loss, train_scores, train_score_cost = self._train_objective( configuration, fidelity, shuffle, rng, evaluation="val" ) @@ -244,6 +242,7 @@ def _mo_objective_function(self, val_loss = 1 - val_scores["acc"] val_sensitivity = (val_scores['f1'] * val_scores['precision']) / (2 * val_scores['precision'] - val_scores['f1'] ) val_scores['fnr'] = 1 - val_sensitivity + val_scores['fpr'] = 1 - val_scores['precision'] test_scores = dict() test_score_cost = dict() @@ -255,6 +254,7 @@ def _mo_objective_function(self, test_sensitivity = (test_scores['f1'] * test_scores['precision']) / (2 * test_scores['precision'] - test_scores['f1'] ) test_scores['fnr'] = 1 - test_sensitivity + test_scores['fpr'] = 1 - test_scores['precision'] info = { 'train_loss': train_loss, @@ -276,8 +276,9 @@ def _mo_objective_function(self, 'function_value': { # The original benchmark returned the accuracy with range [0, 100]. # We cast it to a minimization problem with range [0-1] to have a more standardized return value. - 'val_loss': val_loss, - 'fnr': val_scores['fnr'], + 'val_loss': float(val_loss), + 'fnr': float(val_scores['fnr']), + 'fpr': float(val_scores['fpr']), }, 'cost': model_fit_time + info['val_costs']['acc'], @@ -308,7 +309,7 @@ def _mo_objective_function_test(self, test_sensitivity = (test_scores['f1'] * test_scores['precision']) / (2 * test_scores['precision'] - test_scores['f1'] ) test_scores['fnr'] = 1 - test_sensitivity - + test_scores['fpr'] = 1 - test_scores['precision'] info = { 'train_loss': train_loss, 'val_loss': None, @@ -331,9 +332,9 @@ def _mo_objective_function_test(self, # We cast it to a minimization problem with range [0-1] to have a more standardized return value. 'test_loss': float(info['test_loss']), # 'f1': test_scores['f1'], - # 'precision': test_scores['precision'], + 'fpr': float(test_scores['fpr']), # 'balanced_accuracy': test_scores['bal_acc'], - 'fnr': test_scores['fnr'], + 'fnr': float(test_scores['fnr']), }, 'cost': float(model_fit_time + info['test_costs']['acc']), 'info': info @@ -406,7 +407,7 @@ def objective_function(self, shuffle: bool = False, rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: - return self._mo_objective_function(configuration=configuration, fidelity=fidelity,shuffle=shuffle, rng=rng, + return self._mo_objective_function(configuration=configuration, fidelity=fidelity, shuffle=shuffle, rng=rng, **kwargs) @AbstractMultiObjectiveBenchmark.check_parameters @@ -420,5 +421,5 @@ def objective_function_test(self, **kwargs) @staticmethod def get_objective_names() -> List[str]: - return ['val_loss', 'fnr', bal_acc', 'f1', 'precision'] + return ['val_loss', 'fnr', 'fpr'] From bdf31ecc5e72d3063252e7e45626f99158340333 Mon Sep 17 00:00:00 2001 From: ayushi-3536 Date: Wed, 10 May 2023 01:50:44 +0200 Subject: [PATCH 09/17] add debug log --- hpobench/benchmarks/ml/tabular_benchmark.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hpobench/benchmarks/ml/tabular_benchmark.py b/hpobench/benchmarks/ml/tabular_benchmark.py index c7585ad7..80a57e0f 100644 --- a/hpobench/benchmarks/ml/tabular_benchmark.py +++ b/hpobench/benchmarks/ml/tabular_benchmark.py @@ -134,6 +134,7 @@ def _search_dataframe(self, row_dict, df): result = result.filter(regex='result.') #convert the result to dict resultdict = result.to_dict() + print(f'resultdict: {resultdict}') result={} for key, value in resultdict.items(): keys = key.split('.') @@ -143,6 +144,7 @@ def _search_dataframe(self, row_dict, df): d[k] = {} d = d[k] d[keys[-1]] = value + print(f'result: {result}') return result['result'] From 6f0fc832b0472db016ee95cd2b1414b0fb1d29c9 Mon Sep 17 00:00:00 2001 From: Ayushi Date: Wed, 10 May 2023 07:28:50 +0200 Subject: [PATCH 10/17] fix cost calculation --- hpobench/benchmarks/ml/tabular_benchmark.py | 35 ++++++++++++++------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/hpobench/benchmarks/ml/tabular_benchmark.py b/hpobench/benchmarks/ml/tabular_benchmark.py index 80a57e0f..d0f477ef 100644 --- a/hpobench/benchmarks/ml/tabular_benchmark.py +++ b/hpobench/benchmarks/ml/tabular_benchmark.py @@ -124,6 +124,20 @@ def get_fidelity_range(self) -> List: def _search_dataframe(self, row_dict, df): # https://stackoverflow.com/a/46165056/8363967 mask = np.array([True] * df.shape[0]) + #for some benchmarks result is comprehensive in the form of a dict + #for some the dictionary is flattened and column name is set by joining keys using delimiter '.' + print("df", df) + if 'result' in df.columns: + for i, param in enumerate(df.drop("result", axis=1).columns): + mask *= df[param].values == row_dict[param] + idx = np.where(mask) + assert len(idx) == 1, 'The query has resulted into mulitple matches. This should not happen. ' \ + f'The Query was {row_dict}' + idx = idx[0][0] + result = df.iloc[idx]["result"] + return result + + for i, param in enumerate(df.drop(df.filter(regex='result.').columns, axis=1)): mask *= df[param].values == row_dict[param] idx = np.where(mask) @@ -134,7 +148,6 @@ def _search_dataframe(self, row_dict, df): result = result.filter(regex='result.') #convert the result to dict resultdict = result.to_dict() - print(f'resultdict: {resultdict}') result={} for key, value in resultdict.items(): keys = key.split('.') @@ -144,7 +157,6 @@ def _search_dataframe(self, row_dict, df): d[k] = {} d = d[k] d[keys[-1]] = value - print(f'result: {result}') return result['result'] @@ -178,17 +190,16 @@ def _moobjective( - for metric_key, metric_value in metrics.items(): - for seed in seeds: - key_path["seed"] = seed - res = self._search_dataframe(key_path, self.table) + for seed in seeds: + key_path["seed"] = seed + res = self._search_dataframe(key_path, self.table) + costs["model_cost"] = costs["model_cost"] + res["info"]["model_cost"] + for metric_key, metric_value in metrics.items(): dict_metrics[metric_key].append(res["info"][score_key][metric_key]) - costs["model_cost"] = costs["model_cost"] + res["info"]["model_cost"] costs[metric_key] = costs[metric_key] + res["info"][cost_key][metric_key] - info[seed] = res["info"] - key_path.pop("seed") + info[seed] = res["info"] + key_path.pop("seed") - result_dict = { 'function_value': { # The original benchmark returned the accuracy with range [0, 100]. @@ -219,7 +230,10 @@ def objective_function(self, configuration: Union[CS.Configuration, Dict], seed: Union[int, None] = None, **kwargs) -> Dict: result_dict = self._moobjective(config=configuration, fidelity=fidelity, seed=seed, evaluation="val") + + sensitivity = (result_dict['function_value']['f1'] * result_dict['function_value']['precision']) / (2 * result_dict['function_value']['precision'] - result_dict['function_value']['f1'] ) + false_negative_rate = 1 - sensitivity false_positive_rate = 1 - result_dict['function_value']['precision'] #not considering cost for loss as it is not one of the objectives @@ -234,7 +248,6 @@ def objective_function(self, configuration: Union[CS.Configuration, Dict], 'cost': cost, 'info': result_dict['info'] } - return result @AbstractMultiObjectiveBenchmark.check_parameters From d5d3b2336c3b8489f5c56f043472726148715e77 Mon Sep 17 00:00:00 2001 From: Ayushi Date: Wed, 10 May 2023 08:08:36 +0200 Subject: [PATCH 11/17] remove unnecessary logs --- hpobench/benchmarks/ml/tabular_benchmark.py | 4 ++-- hpobench/container/benchmarks/ml/__init__.py | 4 ++-- hpobench/container/benchmarks/ml/tabular_benchmark.py | 10 +++++++++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/hpobench/benchmarks/ml/tabular_benchmark.py b/hpobench/benchmarks/ml/tabular_benchmark.py index d0f477ef..e1bc9a64 100644 --- a/hpobench/benchmarks/ml/tabular_benchmark.py +++ b/hpobench/benchmarks/ml/tabular_benchmark.py @@ -126,7 +126,7 @@ def _search_dataframe(self, row_dict, df): mask = np.array([True] * df.shape[0]) #for some benchmarks result is comprehensive in the form of a dict #for some the dictionary is flattened and column name is set by joining keys using delimiter '.' - print("df", df) + if 'result' in df.columns: for i, param in enumerate(df.drop("result", axis=1).columns): mask *= df[param].values == row_dict[param] @@ -210,7 +210,7 @@ def _moobjective( 'bal_acc': float(np.mean(dict_metrics['bal_acc'])), }, 'cost': { - 'acc': float(costs['acc'] / len(seeds)), + 'acc': float(costs['acc'] / len(seeds)), 'precision': float(costs['precision'] / len(seeds)), 'f1': float(costs['f1'] / len(seeds)), 'bal_acc': float(costs['bal_acc'] / len(seeds)), diff --git a/hpobench/container/benchmarks/ml/__init__.py b/hpobench/container/benchmarks/ml/__init__.py index ed2ce40f..ddc66a34 100644 --- a/hpobench/container/benchmarks/ml/__init__.py +++ b/hpobench/container/benchmarks/ml/__init__.py @@ -4,7 +4,7 @@ from hpobench.container.benchmarks.ml.rf_benchmark import RandomForestBenchmark, RandomForestBenchmarkBB, \ RandomForestBenchmarkMF from hpobench.container.benchmarks.ml.svm_benchmark import SVMBenchmark, SVMBenchmarkBB, SVMBenchmarkMF -from hpobench.container.benchmarks.ml.tabular_benchmark import TabularBenchmark +from hpobench.container.benchmarks.ml.tabular_benchmark import TabularBenchmark, TabularBenchmarkMO from hpobench.container.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark, XGBoostBenchmarkBB, XGBoostBenchmarkMF @@ -13,5 +13,5 @@ 'NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF', 'RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF', 'SVMBenchmark', 'SVMBenchmarkBB', 'SVMBenchmarkMF', - 'TabularBenchmark', + 'TabularBenchmark', 'TabularBenchmarkMO', 'XGBoostBenchmark', 'XGBoostBenchmarkBB', 'XGBoostBenchmarkMF'] diff --git a/hpobench/container/benchmarks/ml/tabular_benchmark.py b/hpobench/container/benchmarks/ml/tabular_benchmark.py index 6d19953b..d2eb5361 100644 --- a/hpobench/container/benchmarks/ml/tabular_benchmark.py +++ b/hpobench/container/benchmarks/ml/tabular_benchmark.py @@ -13,5 +13,13 @@ def __init__(self, **kwargs): kwargs['latest'] = kwargs.get('container_tag', '0.0.1') super(TabularBenchmark, self).__init__(**kwargs) +class TabularBenchmarkMO(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'TabularBenchmarkMO') + kwargs['container_name'] = kwargs.get('container_name', 'mo_ml_mmfb') + kwargs['container_source'] = 'oras://gitlab.tf.uni-freiburg.de:5050/sharmaa/hpobench-registry' + kwargs['container_tag'] = '0.0.8' + kwargs['latest'] = '0.0.8' + super(TabularBenchmarkMO, self).__init__(**kwargs) -__all__ = ['TabularBenchmark'] +__all__ = ['TabularBenchmark', 'TabularBenchmarkMO'] From 9a9af7bb405a617978506d34f1cf9037114d931f Mon Sep 17 00:00:00 2001 From: ayushi-3536 Date: Fri, 12 May 2023 16:11:08 +0200 Subject: [PATCH 12/17] added container changes for development --- hpobench/benchmarks/ml/tabular_benchmark.py | 14 ++++++++++---- hpobench/container/benchmarks/ml/__init__.py | 4 ++-- hpobench/container/benchmarks/ml/lr_benchmark.py | 8 ++++++-- hpobench/container/benchmarks/ml/nn_benchmark.py | 10 ++++++++-- hpobench/container/benchmarks/ml/rf_benchmark.py | 10 ++++++++-- hpobench/container/benchmarks/ml/svm_benchmark.py | 9 +++++++-- .../container/benchmarks/ml/tabular_benchmark.py | 13 ++++++++++++- .../container/benchmarks/ml/xgboost_benchmark.py | 6 ++++-- hpobench/util/data_manager.py | 2 +- 9 files changed, 58 insertions(+), 18 deletions(-) diff --git a/hpobench/benchmarks/ml/tabular_benchmark.py b/hpobench/benchmarks/ml/tabular_benchmark.py index 80a57e0f..f86ed873 100644 --- a/hpobench/benchmarks/ml/tabular_benchmark.py +++ b/hpobench/benchmarks/ml/tabular_benchmark.py @@ -42,6 +42,7 @@ def __init__(self, self.dm = TabularDataManager(model, task_id, data_dir) self.table, self.metadata = self.dm.load() + self.exp_args = self.metadata["exp_args"] self.config_spaces = self.metadata["config_spaces"] self.global_minimums = self.metadata["global_min"] @@ -124,6 +125,15 @@ def get_fidelity_range(self) -> List: def _search_dataframe(self, row_dict, df): # https://stackoverflow.com/a/46165056/8363967 mask = np.array([True] * df.shape[0]) + if "result" in df.columns: + for i, param in enumerate(df.drop("result", axis=1).columns): + mask *= df[param].values == row_dict[param] + idx = np.where(mask) + assert len(idx) == 1, 'The query has resulted into mulitple matches. This should not happen. ' \ + f'The Query was {row_dict}' + idx = idx[0][0] + result = df.iloc[idx]["result"] + return result for i, param in enumerate(df.drop(df.filter(regex='result.').columns, axis=1)): mask *= df[param].values == row_dict[param] idx = np.where(mask) @@ -134,7 +144,6 @@ def _search_dataframe(self, row_dict, df): result = result.filter(regex='result.') #convert the result to dict resultdict = result.to_dict() - print(f'resultdict: {resultdict}') result={} for key, value in resultdict.items(): keys = key.split('.') @@ -144,7 +153,6 @@ def _search_dataframe(self, row_dict, df): d[k] = {} d = d[k] d[keys[-1]] = value - print(f'result: {result}') return result['result'] @@ -187,7 +195,6 @@ def _moobjective( costs[metric_key] = costs[metric_key] + res["info"][cost_key][metric_key] info[seed] = res["info"] key_path.pop("seed") - result_dict = { 'function_value': { @@ -234,7 +241,6 @@ def objective_function(self, configuration: Union[CS.Configuration, Dict], 'cost': cost, 'info': result_dict['info'] } - return result @AbstractMultiObjectiveBenchmark.check_parameters diff --git a/hpobench/container/benchmarks/ml/__init__.py b/hpobench/container/benchmarks/ml/__init__.py index ed2ce40f..7a986208 100644 --- a/hpobench/container/benchmarks/ml/__init__.py +++ b/hpobench/container/benchmarks/ml/__init__.py @@ -4,7 +4,7 @@ from hpobench.container.benchmarks.ml.rf_benchmark import RandomForestBenchmark, RandomForestBenchmarkBB, \ RandomForestBenchmarkMF from hpobench.container.benchmarks.ml.svm_benchmark import SVMBenchmark, SVMBenchmarkBB, SVMBenchmarkMF -from hpobench.container.benchmarks.ml.tabular_benchmark import TabularBenchmark +from hpobench.container.benchmarks.ml.tabular_benchmark import TabularBenchmark, TabularBenchmarkMO from hpobench.container.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark, XGBoostBenchmarkBB, XGBoostBenchmarkMF @@ -13,5 +13,5 @@ 'NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF', 'RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF', 'SVMBenchmark', 'SVMBenchmarkBB', 'SVMBenchmarkMF', - 'TabularBenchmark', + 'TabularBenchmark', 'TabularBenchmarkMO' 'XGBoostBenchmark', 'XGBoostBenchmarkBB', 'XGBoostBenchmarkMF'] diff --git a/hpobench/container/benchmarks/ml/lr_benchmark.py b/hpobench/container/benchmarks/ml/lr_benchmark.py index e07b1f79..0d8ddb86 100644 --- a/hpobench/container/benchmarks/ml/lr_benchmark.py +++ b/hpobench/container/benchmarks/ml/lr_benchmark.py @@ -16,8 +16,12 @@ def __init__(self, **kwargs): class LRBenchmarkMO(AbstractBenchmarkClient): def __init__(self, **kwargs): kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'LRBenchmarkMO') - kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') - kwargs['latest'] = kwargs.get('container_tag', '0.0.2') + kwargs['container_name'] = kwargs.get('container_name', 'mo_ml_mmfb') + kwargs['container_source'] = 'oras://gitlab.tf.uni-freiburg.de:5050/sharmaa/hpobench-registry' + kwargs['container_tag'] = '0.0.6' + kwargs['latest'] = '0.0.6' + # kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') + # kwargs['latest'] = kwargs.get('container_tag', '0.0.2') super(LRBenchmarkMO, self).__init__(**kwargs) diff --git a/hpobench/container/benchmarks/ml/nn_benchmark.py b/hpobench/container/benchmarks/ml/nn_benchmark.py index 8b96f946..92969393 100644 --- a/hpobench/container/benchmarks/ml/nn_benchmark.py +++ b/hpobench/container/benchmarks/ml/nn_benchmark.py @@ -16,8 +16,14 @@ def __init__(self, **kwargs): class NNBenchmarkMO(AbstractBenchmarkClient): def __init__(self, **kwargs): kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'NNBenchmarkMO') - kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') - kwargs['latest'] = kwargs.get('container_tag', '0.0.2') + kwargs['container_name'] = kwargs.get('container_name', 'mo_ml_mmfb') + + #Need to be changed to the production registry + kwargs['container_source'] = 'oras://gitlab.tf.uni-freiburg.de:5050/sharmaa/hpobench-registry' + kwargs['container_tag'] = '0.0.6' + kwargs['latest'] = '0.0.6' + # kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') + # kwargs['latest'] = kwargs.get('container_tag', '0.0.2') super(NNBenchmarkMO, self).__init__(**kwargs) class NNBenchmarkBB(AbstractBenchmarkClient): diff --git a/hpobench/container/benchmarks/ml/rf_benchmark.py b/hpobench/container/benchmarks/ml/rf_benchmark.py index a3ca46ba..9daa324d 100644 --- a/hpobench/container/benchmarks/ml/rf_benchmark.py +++ b/hpobench/container/benchmarks/ml/rf_benchmark.py @@ -18,8 +18,14 @@ def __init__(self, **kwargs): class RandomForestBenchmarkMO(AbstractBenchmarkClient): def __init__(self, **kwargs): kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'RandomForestBenchmarkMO') - kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') - kwargs['latest'] = kwargs.get('container_tag', '0.0.2') + kwargs['container_name'] = kwargs.get('container_name', 'mo_ml_mmfb') + + #Need to be changed to the production registry + kwargs['container_source'] = 'oras://gitlab.tf.uni-freiburg.de:5050/sharmaa/hpobench-registry' + kwargs['container_tag'] = '0.0.6' + kwargs['latest'] = '0.0.6' + # kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') + # kwargs['latest'] = kwargs.get('container_tag', '0.0.2') super(RandomForestBenchmarkMO, self).__init__(**kwargs) diff --git a/hpobench/container/benchmarks/ml/svm_benchmark.py b/hpobench/container/benchmarks/ml/svm_benchmark.py index 46c951a8..8bb1a704 100644 --- a/hpobench/container/benchmarks/ml/svm_benchmark.py +++ b/hpobench/container/benchmarks/ml/svm_benchmark.py @@ -16,8 +16,13 @@ def __init__(self, **kwargs): class SVMBenchmarkMO(AbstractBenchmarkClient): def __init__(self, **kwargs): kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'SVMBenchmarkMO') - kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') - kwargs['latest'] = kwargs.get('container_tag', '0.0.2') + kwargs['container_name'] = kwargs.get('container_name', 'mo_ml_mmfb') + + #Need to be changed to the production registry + kwargs['container_source'] = 'oras://gitlab.tf.uni-freiburg.de:5050/sharmaa/hpobench-registry' + kwargs['container_tag'] = '0.0.6' + kwargs['latest'] = '0.0.6' + #kwargs['latest'] = kwargs.get('container_tag', '0.0.2') super(SVMBenchmarkMO, self).__init__(**kwargs) diff --git a/hpobench/container/benchmarks/ml/tabular_benchmark.py b/hpobench/container/benchmarks/ml/tabular_benchmark.py index 6d19953b..1ea873be 100644 --- a/hpobench/container/benchmarks/ml/tabular_benchmark.py +++ b/hpobench/container/benchmarks/ml/tabular_benchmark.py @@ -13,5 +13,16 @@ def __init__(self, **kwargs): kwargs['latest'] = kwargs.get('container_tag', '0.0.1') super(TabularBenchmark, self).__init__(**kwargs) +class TabularBenchmarkMO(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'TabularBenchmarkMO') + + + #Need to be changed to the production registry + kwargs['container_name'] = kwargs.get('container_name', 'mo_ml_mmfb') + kwargs['container_source'] = 'oras://gitlab.tf.uni-freiburg.de:5050/sharmaa/hpobench-registry' + kwargs['container_tag'] = '0.0.8' + kwargs['latest'] = '0.0.8' + super(TabularBenchmarkMO, self).__init__(**kwargs) -__all__ = ['TabularBenchmark'] +__all__ = ['TabularBenchmark', 'TabularBenchmarkMO'] diff --git a/hpobench/container/benchmarks/ml/xgboost_benchmark.py b/hpobench/container/benchmarks/ml/xgboost_benchmark.py index af1e99cc..644649df 100644 --- a/hpobench/container/benchmarks/ml/xgboost_benchmark.py +++ b/hpobench/container/benchmarks/ml/xgboost_benchmark.py @@ -17,9 +17,11 @@ class XGBoostBenchmarkMO(AbstractBenchmarkClient): def __init__(self, **kwargs): kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'XGBoostBenchmarkMO') kwargs['container_name'] = kwargs.get('container_name', 'mo_ml_mmfb') + + #Need to be changed to the production registry kwargs['container_source'] = 'oras://gitlab.tf.uni-freiburg.de:5050/sharmaa/hpobench-registry' - kwargs['container_tag'] = '0.0.2' - kwargs['latest'] = '0.0.2' #kwargs.get('container_tag', '0.0.2') + kwargs['container_tag'] = '0.0.6' + kwargs['latest'] = '0.0.6' #kwargs.get('container_tag', '0.0.2') super(XGBoostBenchmarkMO, self).__init__(**kwargs) diff --git a/hpobench/util/data_manager.py b/hpobench/util/data_manager.py index c72305e1..9140002a 100644 --- a/hpobench/util/data_manager.py +++ b/hpobench/util/data_manager.py @@ -1172,7 +1172,7 @@ def get_workclass(x): class TabularDataManager(DataManager): - def __init__(self, model: str, task_id: [int, str], data_dir: [str, Path, None] = None): + def __init__(self, model: str, task_id: Union[int, str], data_dir: Union[str, Path, None] = None): super(TabularDataManager, self).__init__() url_dict = dict( From 468896edb93c5642fd3b1c1f55de95ceb47d5228 Mon Sep 17 00:00:00 2001 From: Jake Robertson Date: Wed, 31 May 2023 13:13:55 +0200 Subject: [PATCH 13/17] Added JAHS extra requirements for container creation --- extra_requirements/jahs_bench_201.json | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 extra_requirements/jahs_bench_201.json diff --git a/extra_requirements/jahs_bench_201.json b/extra_requirements/jahs_bench_201.json new file mode 100644 index 00000000..fc220155 --- /dev/null +++ b/extra_requirements/jahs_bench_201.json @@ -0,0 +1,3 @@ +{ + "jahs_bench_201": ["jahs_bench_201@git+https://github.com/automl/jahs_bench_201.git"] +} From bdc7fcb3e9fa2f17ab1eedff05aee7d3f68ef778 Mon Sep 17 00:00:00 2001 From: Jake Robertson Date: Thu, 1 Jun 2023 14:23:05 +0200 Subject: [PATCH 14/17] Fix data loading path error --- hpobench/benchmarks/nas/jahs_benchmarks.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/hpobench/benchmarks/nas/jahs_benchmarks.py b/hpobench/benchmarks/nas/jahs_benchmarks.py index e787fa7c..ba9096da 100755 --- a/hpobench/benchmarks/nas/jahs_benchmarks.py +++ b/hpobench/benchmarks/nas/jahs_benchmarks.py @@ -78,15 +78,6 @@ def download_and_extract_url(url, save_dir, filename): with tarfile.open(save_tar_file, 'r') as f: f.extractall(path=save_dir) - if save_tar_file.name == 'assembled_surrogates.tar': - from shutil import move - _dir = save_dir / 'assembled_surrogates' - _dir.mkdir(exist_ok=True, parents=True) - for dir_name in ['cifar10', 'colorectal_histology', 'fashion_mnist']: - _old_dir = save_dir / dir_name - _new_dir = _dir / dir_name - move(_old_dir, _new_dir) - logger.info("Done extracting") finish_flag.touch() From 30435bfe2d3a30675e8cb8dd70142d72c3a0a3eb Mon Sep 17 00:00:00 2001 From: Jake Robertson Date: Thu, 6 Jul 2023 14:28:32 +0200 Subject: [PATCH 15/17] Update README.md --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 96dd406d..7c77ce44 100755 --- a/README.md +++ b/README.md @@ -45,6 +45,22 @@ fs = b.get_fidelity_space(seed=1) meta = b.get_meta_information() ``` +Multi-Objective benchmarks can be queried in a similar manner + +```python +from hpobench.container.benchmarks.mo.adult_benchmark import AdultBenchmark +b = AdultBenchmark(rng=1) +config = b.get_configuration_space(seed=1).sample_configuration() +result_dict = b.objective_function(configuration=config, fidelity={"budget": 66}, rng=1) +result_dict = b.objective_function(configuration=config, rng=1) + +>>> print(result_dict['function_values']) +{'misclassification_rate': 0.16572832429112494, + 'DSO': 0.197765453723867, + 'DEO': 0.1595593763542093, + 'DFP': 0.10465117283454546} +``` + ## Installation We recommend using a virtual environment. To install HPOBench, please run the following: From cd4ffc98442c8a32ea02cca0d70096dfbb96f5a2 Mon Sep 17 00:00:00 2001 From: Jake Robertson Date: Wed, 12 Jul 2023 11:06:44 +0200 Subject: [PATCH 16/17] Update client_abstract_benchmark.py Changed gitlab source to refer to Jake's registry --- hpobench/container/client_abstract_benchmark.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hpobench/container/client_abstract_benchmark.py b/hpobench/container/client_abstract_benchmark.py index 163a3eff..722ffed1 100755 --- a/hpobench/container/client_abstract_benchmark.py +++ b/hpobench/container/client_abstract_benchmark.py @@ -183,7 +183,7 @@ def load_benchmark(self, benchmark_name: str, container_name: str, container_sou self.container_source = container_source or self.config.container_source self.container_dir = Path(self.config.container_dir) - if (self.container_source.startswith('oras://gitlab.tf.uni-freiburg.de:5050/muelleph/hpobench-registry') + if (self.container_source.startswith('oras://gitlab.tf.uni-freiburg.de:5050/robertsj/mo-hpobench') and container_tag == 'latest'): assert 'latest' in kwargs, 'If the container is hosted on the gitlab registry, make sure that in the ' \ 'container init, the field \'latest\' is set.' @@ -226,7 +226,7 @@ def download_container(container_dir, container_name, container_source, containe # "tag" a new entry in the registry. This might change in the future. But as long as we don't have # a fix for this, we need to map the container tag differently. if (container_source.startswith('oras://gitlab.tf.uni-freiburg.de:5050/muelleph/hpobench-registry')) or \ - (container_source.startswith('oras://gitlab.tf.uni-freiburg.de:5050/sharmaa/hpobench-registry')): + (container_source.startswith('oras://gitlab.tf.uni-freiburg.de:5050/robertsj/mo-hpobench')): cmd += f'{container_source}/{container_name.lower()}/{container_tag}:latest' else: cmd += f'{container_source}/{container_name.lower()}:{container_tag}' From 5706c5c62a3fb4b00460d5102edc139bb55feef2 Mon Sep 17 00:00:00 2001 From: Jake Robertson Date: Mon, 17 Jul 2023 09:57:24 +0200 Subject: [PATCH 17/17] Update config.py Changed default container source to Jake's registry --- hpobench/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hpobench/config.py b/hpobench/config.py index 9d7964e0..81143257 100755 --- a/hpobench/config.py +++ b/hpobench/config.py @@ -65,7 +65,7 @@ def __init__(self): # Options for the singularity container self.socket_dir = Path(self.socket_dir).expanduser().absolute() self.container_dir = self.cache_dir / f'hpobench-{os.getuid()}' - self.container_source = 'oras://gitlab.tf.uni-freiburg.de:5050/muelleph/hpobench-registry' + self.container_source = 'oras://gitlab.tf.uni-freiburg.de:5050/robertsj/mo-hpobench' self.pyro_connect_max_wait = 400 # Read in the hpobenchrc file and set the default values if not specified