Merge pull request #278 from KernelTuner/searchspace_experiments

Extended searchspace construction and input format support

Author: benvanwerkhoven

.gitignore

@@ -2,6 +2,8 @@
 poetry.lock
 noxenv.txt
 noxsettings.toml
+hyperparamtuning/
+*.prof
 
 ### Python ###
 *.pyc
@@ -16,6 +18,8 @@ push_to_pypi.sh
 .nfs*
 *.log
 *.json
+!kernel_tuner/schema/T1/1.0.0/input-schema.json
+!test/test_T1_input.json
 *.csv
 .cache
 *.ipynb_checkpoints

CHANGELOG.md

@@ -3,13 +3,17 @@ All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
+<!-- ## [1.1.0] - 2025 ?? -->
+- Additional improvements to search space construction
 - changed HIP python bindings from pyhip-interface to the official hip-python
+- Added Python 3.13 and experimental 3.14 support
+- Dropped Python 3.8 and 3.9 support (due to incompatibility with newer scipy versions)
 
 ## [1.0.0] - 2024-04-04
 - HIP backend to support tuning HIP kernels on AMD GPUs
 - Experimental features for mixed-precision and accuracy tuning
 - Experimental features for OpenACC tuning
-- Major speedup due to new parser and using revamped python-constraint for searchspace building
+- Major speedup due to new parser and using revamped python-constraint for search space construction
 - Implemented ability to use `PySMT` and `ATF` for searchspace building
 - Added Poetry for dependency and build management
 - Switched from `setup.py` and `setup.cfg` to `pyproject.toml` for centralized metadata, added relevant tests

INSTALL.rst

@@ -20,7 +20,7 @@ Linux users could type the following to download and install Python 3 using Mini
     wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
     bash Miniconda3-latest-Linux-x86_64.sh
 
-You are of course also free to use your own Python installation, and the Kernel Tuner is developed to be fully compatible with Python 3.9 and newer.
+You are of course also free to use your own Python installation, and the Kernel Tuner is developed to be fully compatible with Python 3.10 and newer.
 
 Installing Python Packages
 --------------------------

doc/requirements_test.txt

@@ -27,8 +27,8 @@ Steps with :bash:`sudo` access (e.g. on a local device):
     * After installation, restart your shell. 
 #. Install the required Python versions: 
     * On some systems, additional packages may be needed to build Python versions. For example on Ubuntu: :bash:`sudo apt install build-essential zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libreadline-dev libffi-dev libsqlite3-dev wget libbz2-dev liblzma-dev lzma`.
-    * Install the Python versions with: :bash:`pyenv install 3.9 3.10 3.11 3.12`. The reason we're installing all these versions as opposed to just one, is so we can test against all supported Python versions.
-#. Set the Python versions so they can be found: :bash:`pyenv local 3.9 3.10 3.11 3.12` (replace :bash:`local` with :bash:`global` when not using the virtualenv).
+    * Install the Python versions with: :bash:`pyenv install 3.9 3.10 3.11 3.12 3.13`. The reason we're installing all these versions as opposed to just one, is so we can test against all supported Python versions.
+#. Set the Python versions so they can be found: :bash:`pyenv local 3.9 3.10 3.11 3.12 3.13` (replace :bash:`local` with :bash:`global` when not using the virtualenv).
 #. Setup a local virtual environment in the folder: :bash:`pyenv virtualenv 3.11 kerneltuner` (or whatever environment name and Python version you prefer).
 #. `Install Poetry <https://python-poetry.org/docs/#installing-with-the-official-installer>`__. 
     * Use :bash:`curl -sSL https://install.python-poetry.org | python3 -` to install Poetry.

doc/source/dev-environment.rst

@@ -1,5 +1,5 @@
 from kernel_tuner.integration import store_results, create_device_targets
-from kernel_tuner.interface import tune_kernel, run_kernel
+from kernel_tuner.interface import tune_kernel, tune_kernel_T1, run_kernel
 
 from importlib.metadata import version
 

kernel_tuner/__init__.py

@@ -1,16 +1,16 @@
-"""This module contains the interface of all kernel_tuner backends"""
+"""This module contains the interface of all kernel_tuner backends."""
 from __future__ import print_function
 
 from abc import ABC, abstractmethod
 
 
 class Backend(ABC):
-    """Base class for kernel_tuner backends"""
+    """Base class for kernel_tuner backends."""
 
     @abstractmethod
     def ready_argument_list(self, arguments):
         """This method must implement the allocation of the arguments on device memory."""
-        pass
+        return arguments
 
     @abstractmethod
     def compile(self, kernel_instance):
@@ -64,7 +64,7 @@ def refresh_memory(self, device_memory, host_arguments, should_sync):
 
 
 class GPUBackend(Backend):
-    """Base class for GPU backends"""
+    """Base class for GPU backends."""
 
     @abstractmethod
     def __init__(self, device, iterations, compiler_options, observers):
@@ -93,7 +93,7 @@ def refresh_memory(self, gpu_memory, host_arguments, should_sync):
 
 
 class CompilerBackend(Backend):
-    """Base class for compiler backends"""
+    """Base class for compiler backends."""
 
     @abstractmethod
     def __init__(self, iterations, compiler_options, compiler):

kernel_tuner/backends/backend.py

@@ -0,0 +1,131 @@
+"""This module contains a 'device' for hyperparameter tuning using the autotuning methodology."""
+
+import platform
+from pathlib import Path
+
+from numpy import mean
+
+from kernel_tuner.backends.backend import Backend
+from kernel_tuner.observers.observer import BenchmarkObserver
+
+try:
+    methodology_available = True
+    from autotuning_methodology.experiments import generate_experiment_file
+    from autotuning_methodology.report_experiments import get_strategy_scores
+except ImportError:
+    methodology_available = False
+
+
+class ScoreObserver(BenchmarkObserver):
+    def __init__(self, dev):
+        self.dev = dev
+        self.scores = []
+
+    def after_finish(self):
+        self.scores.append(self.dev.last_score)
+
+    def get_results(self):
+        results = {'score': mean(self.scores), 'scores': self.scores.copy()}
+        self.scores = []
+        return results
+
+class HypertunerFunctions(Backend):
+    """Class for executing hyperparameter tuning."""
+    units = {}
+
+    def __init__(self, iterations):
+        self.iterations = iterations
+        self.observers = [ScoreObserver(self)]
+        self.name = platform.processor()
+        self.max_threads = 1024
+        self.last_score = None
+
+        # set the environment options
+        env = dict()
+        env["iterations"] = self.iterations
+        self.env = env
+
+        # check for the methodology package
+        if methodology_available is not True:
+            raise ImportError("Unable to import the autotuning methodology, run `pip install autotuning_methodology`.")
+
+    def ready_argument_list(self, arguments):
+        arglist = super().ready_argument_list(arguments)
+        if arglist is None:
+            arglist = []
+        return arglist
+    
+    def compile(self, kernel_instance):
+        super().compile(kernel_instance)
+        path = Path(__file__).parent.parent.parent / "hyperparamtuning"
+        path.mkdir(exist_ok=True)
+
+        # TODO get applications & GPUs args from benchmark
+        gpus = ["RTX_3090", "RTX_2080_Ti"]
+        applications = None
+        # applications = [
+        #     {
+        #         "name": "convolution",
+        #         "folder": "./cached_data_used/kernels",
+        #         "input_file": "convolution.json"
+        #     },
+        #     {
+        #         "name": "pnpoly",
+        #         "folder": "./cached_data_used/kernels",
+        #         "input_file": "pnpoly.json"
+        #     }
+        # ]
+
+        # strategy settings
+        strategy: str = kernel_instance.arguments[0]
+        hyperparams = [{'name': k, 'value': v} for k, v in kernel_instance.params.items()]
+        hyperparams_string = "_".join(f"{k}={str(v)}" for k, v in kernel_instance.params.items())
+        searchspace_strategies = [{
+            "autotuner": "KernelTuner",
+            "name": f"{strategy.lower()}_{hyperparams_string}",
+            "display_name": strategy.replace('_', ' ').capitalize(),
+            "search_method": strategy.lower(),
+            'search_method_hyperparameters': hyperparams
+        }]
+
+        # any additional settings
+        override = { 
+            "experimental_groups_defaults": { 
+                "samples": self.iterations 
+            }
+        }
+
+        name = kernel_instance.name if len(kernel_instance.name) > 0 else kernel_instance.kernel_source.kernel_name
+        experiments_filepath = generate_experiment_file(name, path, searchspace_strategies, applications, gpus, 
+                                                        override=override, overwrite_existing_file=True)
+        return str(experiments_filepath)
+    
+    def start_event(self):
+        return super().start_event()
+    
+    def stop_event(self):
+        return super().stop_event()
+    
+    def kernel_finished(self):
+        super().kernel_finished()
+        return True
+    
+    def synchronize(self):
+        return super().synchronize()
+    
+    def run_kernel(self, func, gpu_args=None, threads=None, grid=None, stream=None):
+        # generate the experiments file
+        experiments_filepath = Path(func)
+
+        # run the methodology to get a fitness score for this configuration
+        scores = get_strategy_scores(str(experiments_filepath))
+        self.last_score = scores[list(scores.keys())[0]]['score']
+    
+    def memset(self, allocation, value, size):
+        return super().memset(allocation, value, size)
+    
+    def memcpy_dtoh(self, dest, src):
+        return super().memcpy_dtoh(dest, src)
+    
+    def memcpy_htod(self, dest, src):
+        return super().memcpy_htod(dest, src)