Skip to content

Commit 5c6cf6c

Browse files
fjwillemsenfjwillemsen
authored
Merge pull request #3 from AutoTuningAssociation/standardization
Standardization, extension of visualizations, benchmark hub, and general updates
2 parents b0075f8 + 0a83604 commit 5c6cf6c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+277624
-1494
lines changed

‎.github/workflows/build-test-python-package.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ jobs:
1717
runs-on: ubuntu-latest
1818
strategy:
1919
matrix:
20-
python-version: ["3.9", "3.10", "3.11"]
20+
python-version: ["3.10", "3.11", "3.12", "3.13"]
2121

2222
steps:
2323
- uses: actions/checkout@v4

‎.github/workflows/publish-package.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,10 @@ jobs:
2323
steps:
2424
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
2525
- uses: actions/checkout@v3
26-
- name: Set up Python 3.9
26+
- name: Set up Python 3.12
2727
uses: actions/setup-python@v4
2828
with:
29-
python-version: 3.9
29+
python-version: 3.12
3030
cache: pip
3131

3232
# Publishes to PyPi

‎.gitignore‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,16 @@ cached_data_used/kernels/*
1010
cached_data_used/visualizations/*
1111
cached_data_used/last_run/*
1212
cached_data_used/import_runs/*
13+
methodology_paper_evaluation/run/*
14+
hyperparametertuning/*
15+
hyperparametertuning_*/*
16+
test_run_experiment/*
17+
18+
# ignore setup files
19+
*/setup/*_input.json
1320

1421
# exceptions
15-
!cached_data_used/cachefiles/ktt_values_to_kerneltuner.py
22+
!cached_data_used/cachefiles/*.py
1623

1724
# ignore specific experiment files
1825
experiment_files/milo.json

‎.gitmodules‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[submodule "benchmark_hub"]
2+
path = benchmark_hub
3+
url = https://github.com/AutoTuningAssociation/benchmark_hub.git

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ Currently, the stable releases of this software package are compatible with [Ker
2222
## Installation
2323
The package can be installed with `pip install autotuning_methodology`.
2424
Alternatively, it can be installed by cloning this repository and running `pip install .` in the root of the cloned project.
25-
Python >= 3.9 is supported.
25+
Like most Python packages, installing in a virtual environment or with `pipx` is recommended. Python >= 3.10 is supported.
2626

2727
## Notable features
2828
- Official software by the authors of the methodology-defining paper.

‎benchmark_hub‎

Submodule benchmark_hub added at ff76e2c

‎cached_data_used/cachefiles/ktt_values_to_kerneltuner.py‎

Lines changed: 0 additions & 116 deletions
This file was deleted.

‎docs/getting_started.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ Getting Started
33

44
Start out by installing the package.
55
The simplest way to do this is ``pip install autotuning_methodology``.
6-
Python 3.9 and up are supported.
6+
Python 3.10 and up are supported.
77

88
Defining an experiment
99
^^^^^^^^^^^^^^^^^^^^^^

‎experiment_files/compare_bo.json‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
{
2+
"version": "1.0.0",
3+
"name": "Methodology paper evaluation",
4+
"parent_folder": "./methodology_paper_evaluation",
5+
"experimental_groups_defaults": {
6+
"applications": [
7+
{
8+
"name": "convolution",
9+
"folder": "./cached_data_used/kernels",
10+
"input_file": "convolution.json"
11+
},
12+
{
13+
"name": "pnpoly",
14+
"folder": "./cached_data_used/kernels",
15+
"input_file": "pnpoly.json"
16+
}
17+
],
18+
"gpus": [
19+
"RTX_3090",
20+
"RTX_2080_Ti"
21+
],
22+
"pattern_for_full_search_space_filenames": {
23+
"regex": "./cached_data_used/cachefiles/${applications}/${gpus}_T4.json"
24+
},
25+
"stochastic": true,
26+
"repeats": 100,
27+
"samples": 32,
28+
"minimum_number_of_valid_search_iterations": 20,
29+
"ignore_cache": false
30+
},
31+
"search_strategies": [
32+
{
33+
"name": "bayes_opt_og",
34+
"search_method": "bayes_opt",
35+
"display_name": "Bayesian Optimization",
36+
"autotuner": "KernelTuner"
37+
},
38+
{
39+
"name": "dual_annealing",
40+
"search_method": "dual_annealing",
41+
"display_name": "Dual Annealing",
42+
"autotuner": "KernelTuner"
43+
}
44+
],
45+
"statistics_settings": {
46+
"minimization": true,
47+
"cutoff_percentile": 0.96,
48+
"cutoff_percentile_start": 0.5,
49+
"cutoff_type": "fevals",
50+
"objective_time_keys": [
51+
"all"
52+
],
53+
"objective_performance_keys": [
54+
"time"
55+
]
56+
},
57+
"visualization_settings": {
58+
"x_axis_value_types": [
59+
"fevals",
60+
"time",
61+
"aggregated"
62+
],
63+
"y_axis_value_types": [
64+
"normalized",
65+
"baseline"
66+
],
67+
"resolution": 1000.0,
68+
"confidence_level": 0.95,
69+
"compare_baselines": false,
70+
"compare_split_times": false
71+
}
72+
}

‎experiment_files/compare_simple.json‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
{
2+
"version": "1.2.0",
3+
"name": "Compare constrained strategies pyATF vs KT",
4+
"parent_folder": "/var/scratch/fjwillem/constrained_optimization",
5+
"experimental_groups_defaults": {
6+
"applications": [
7+
{
8+
"name": "dedispersion_milo",
9+
"folder": "../autotuning_methodology/benchmark_hub/kernels",
10+
"input_file": "dedispersion_milo.json",
11+
"objective_performance_keys": [
12+
"time"
13+
]
14+
}
15+
],
16+
"gpus": [
17+
"A100",
18+
"A4000"
19+
],
20+
"pattern_for_full_search_space_filenames": {
21+
"regex": "./benchmark_hub/cachefiles/${applications}/${gpus}_T4.json"
22+
},
23+
"stochastic": true,
24+
"repeats": 100,
25+
"samples": 32,
26+
"minimum_fraction_of_budget_valid": 0.1,
27+
"minimum_number_of_valid_search_iterations": 10,
28+
"ignore_cache": false
29+
},
30+
"search_strategies": [
31+
{
32+
"name": "genetic_algorithm_constrained",
33+
"search_method": "genetic_algorithm",
34+
"search_method_hyperparameters": [
35+
{
36+
"name": "constraint_aware",
37+
"value": true
38+
}
39+
],
40+
"display_name": "KT GA constrained",
41+
"autotuner": "KernelTuner"
42+
},
43+
{
44+
"name": "genetic_algorithm_non_constrained",
45+
"search_method": "genetic_algorithm",
46+
"search_method_hyperparameters": [
47+
{
48+
"name": "constraint_aware",
49+
"value": false
50+
}
51+
],
52+
"display_name": "KT GA non-constrained",
53+
"autotuner": "KernelTuner",
54+
"color_parent": "genetic_algorithm_constrained"
55+
},
56+
{
57+
"name": "pso_constrained",
58+
"search_method": "pso",
59+
"search_method_hyperparameters": [
60+
{
61+
"name": "constraint_aware",
62+
"value": true
63+
}
64+
],
65+
"display_name": "KT PSO constrained",
66+
"autotuner": "KernelTuner"
67+
},
68+
{
69+
"name": "pso_non_constrained",
70+
"search_method": "pso",
71+
"search_method_hyperparameters": [
72+
{
73+
"name": "constraint_aware",
74+
"value": false
75+
}
76+
],
77+
"display_name": "KT PSO non-constrained",
78+
"autotuner": "KernelTuner",
79+
"color_parent": "pso_constrained"
80+
},
81+
{
82+
"name": "simulated_annealing_constrained",
83+
"search_method": "simulated_annealing",
84+
"search_method_hyperparameters": [
85+
{
86+
"name": "constraint_aware",
87+
"value": true
88+
}
89+
],
90+
"display_name": "KT SA constrained",
91+
"autotuner": "KernelTuner"
92+
},
93+
{
94+
"name": "simulated_annealing_non_constrained",
95+
"search_method": "simulated_annealing",
96+
"search_method_hyperparameters": [
97+
{
98+
"name": "constraint_aware",
99+
"value": false
100+
}
101+
],
102+
"display_name": "KT SA non-constrained",
103+
"autotuner": "KernelTuner",
104+
"color_parent": "simulated_annealing_constrained"
105+
}
106+
],
107+
"statistics_settings": {
108+
"cutoff_percentile": 0.95,
109+
"cutoff_percentile_start": 0.01,
110+
"cutoff_type": "time",
111+
"objective_time_keys": [
112+
"all"
113+
]
114+
},
115+
"visualization_settings": {
116+
"plots": [
117+
{
118+
"scope": "aggregate",
119+
"style": "head2head",
120+
"comparison": {
121+
"unit": "time",
122+
"relative_time": 0.5
123+
},
124+
"annotate": true
125+
}
126+
],
127+
"resolution": 1000.0,
128+
"confidence_level": 0.95,
129+
"compare_baselines": false,
130+
"compare_split_times": false
131+
}
132+
}

0 commit comments

Comments
 (0)