diff --git a/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py b/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py index 8c5e4c24e1e..a90485dfa2c 100644 --- a/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py +++ b/src/main/python/systemds/scuro/drsearch/hyperparameter_tuner.py @@ -103,7 +103,9 @@ def extract_k_best_modalities_per_task(self): representations[task.model.name] = {} for modality in self.modalities: k_best_results, cached_data = ( - self.optimization_results.get_k_best_results(modality, self.k, task) + self.optimization_results.get_k_best_results( + modality, self.k, task, self.scoring_metric + ) ) representations[task.model.name][modality.modality_id] = k_best_results self.k_best_representations[task.model.name].extend(k_best_results) diff --git a/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py b/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py index 9d0088a976a..76831f6aae8 100644 --- a/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py +++ b/src/main/python/systemds/scuro/drsearch/multimodal_optimizer.py @@ -19,6 +19,7 @@ # # ------------------------------------------------------------- import os +import torch import multiprocessing as mp import itertools import threading @@ -83,8 +84,9 @@ def _evaluate_dag_worker(dag_pickle, task_pickle, modalities_pickle, debug=False return OptimizationResult( dag=dag_copy, - train_score=scores[0], - val_score=scores[1], + train_score=scores[0].average_scores, + val_score=scores[1].average_scores, + test_score=scores[2].average_scores, runtime=total_time, task_name=task_copy.model.name, task_time=eval_time, @@ -106,6 +108,7 @@ def __init__( debug: bool = True, min_modalities: int = 2, max_modalities: int = None, + metric: str = "accuracy", ): self.modalities = modalities self.tasks = tasks @@ -116,6 +119,7 @@ def __init__( self.operator_registry = Registry() self.fusion_operators = self.operator_registry.get_fusion_operators() + self.metric_name = metric self.k_best_representations = self._extract_k_best_representations( unimodal_optimization_results @@ -242,7 +246,7 @@ def _extract_k_best_representations( for modality in self.modalities: k_best_results, cached_data = ( unimodal_optimization_results.get_k_best_results( - modality, self.k, task + modality, self.k, task, self.metric_name ) ) @@ -367,6 +371,8 @@ def _evaluate_dag(self, dag: RepresentationDag, task: Task) -> "OptimizationResu task_copy, ) + torch.cuda.empty_cache() + if fused_representation is None: return None @@ -388,8 +394,9 @@ def _evaluate_dag(self, dag: RepresentationDag, task: Task) -> "OptimizationResu return OptimizationResult( dag=dag_copy, - train_score=scores[0], - val_score=scores[1], + train_score=scores[0].average_scores, + val_score=scores[1].average_scores, + test_score=scores[2].average_scores, runtime=total_time, representation_time=total_time - eval_time, task_name=task_copy.model.name, @@ -479,6 +486,7 @@ class OptimizationResult: dag: RepresentationDag train_score: PerformanceMeasure = None val_score: PerformanceMeasure = None + test_score: PerformanceMeasure = None runtime: float = 0.0 task_time: float = 0.0 representation_time: float = 0.0 diff --git a/src/main/python/systemds/scuro/drsearch/ranking.py b/src/main/python/systemds/scuro/drsearch/ranking.py index 831a059eb88..b4b8a392ea1 100644 --- a/src/main/python/systemds/scuro/drsearch/ranking.py +++ b/src/main/python/systemds/scuro/drsearch/ranking.py @@ -19,8 +19,7 @@ # # ------------------------------------------------------------- -from dataclasses import replace -from typing import Callable, Iterable, List, Optional +from typing import Callable, Iterable, Optional def rank_by_tradeoff( @@ -31,7 +30,7 @@ def rank_by_tradeoff( runtime_accessor: Optional[Callable[[object], float]] = None, cache_scores: bool = True, score_attr: str = "tradeoff_score", -) -> List: +): entries = list(entries) if not entries: return [] @@ -39,6 +38,7 @@ def rank_by_tradeoff( performance_score_accessor = lambda entry: getattr(entry, "val_score")[ performance_metric_name ] + if runtime_accessor is None: def runtime_accessor(entry): @@ -77,14 +77,17 @@ def safe_normalize(values, vmin, vmax): if cache_scores: for entry, score in zip(entries, scores): if hasattr(entry, score_attr): - try: - new_entry = replace(entry, **{score_attr: score}) - entries[entries.index(entry)] = new_entry - except TypeError: - setattr(entry, score_attr, score) + setattr(entry, score_attr, score) else: setattr(entry, score_attr, score) - return sorted( - entries, key=lambda entry: getattr(entry, score_attr, 0.0), reverse=True - ) + sorted_entries = sorted(entries, key=lambda e: e.tradeoff_score, reverse=True) + + sorted_indices = [ + i + for i, _ in sorted( + enumerate(entries), key=lambda pair: pair[1].tradeoff_score, reverse=True + ) + ] + + return sorted_entries, sorted_indices diff --git a/src/main/python/systemds/scuro/drsearch/task.py b/src/main/python/systemds/scuro/drsearch/task.py index bfd1f16ab37..fbe08bcc61e 100644 --- a/src/main/python/systemds/scuro/drsearch/task.py +++ b/src/main/python/systemds/scuro/drsearch/task.py @@ -20,12 +20,10 @@ # ------------------------------------------------------------- import copy import time -from typing import List, Union -from systemds.scuro.modality.modality import Modality -from systemds.scuro.representations.representation import Representation +from typing import List from systemds.scuro.models.model import Model import numpy as np -from sklearn.model_selection import KFold +from sklearn.model_selection import train_test_split class PerformanceMeasure: @@ -69,7 +67,8 @@ def __init__( val_indices: List, kfold=5, measure_performance=True, - performance_measures="accuracy", + performance_measures=["accuracy"], + fusion_train_split=0.8, ): """ Parent class for the prediction task that is performed on top of the aligned representation @@ -85,7 +84,7 @@ def __init__( self.model = model self.labels = labels self.train_indices = train_indices - self.val_indices = val_indices + self.test_indices = val_indices self.kfold = kfold self.measure_performance = measure_performance self.inference_time = [] @@ -94,6 +93,47 @@ def __init__( self.performance_measures = performance_measures self.train_scores = PerformanceMeasure("train", performance_measures) self.val_scores = PerformanceMeasure("val", performance_measures) + self.test_scores = PerformanceMeasure("test", performance_measures) + self.fusion_train_indices = None + self._create_cv_splits() + + def _create_cv_splits(self): + train_labels = [self.labels[i] for i in self.train_indices] + train_labels_array = np.array(train_labels) + + train_indices_array = np.array(self.train_indices) + + self.cv_train_indices = [] + self.cv_val_indices = [] + + for fold_idx in range(self.kfold): + fold_train_indices_array, fold_val_indices_array, _, _ = train_test_split( + train_indices_array, + train_labels_array, + test_size=0.2, + shuffle=True, + random_state=11 + fold_idx, + ) + + fold_train_indices = fold_train_indices_array.tolist() + fold_val_indices = fold_val_indices_array.tolist() + + self.cv_train_indices.append(fold_train_indices) + self.cv_val_indices.append(fold_val_indices) + + overlap = set(fold_train_indices) & set(fold_val_indices) + if overlap: + raise ValueError( + f"Fold {fold_idx}: Overlap detected between train and val indices: {overlap}" + ) + + all_val_indices = set() + for val_indices in self.cv_val_indices: + all_val_indices.update(val_indices) + + self.fusion_train_indices = [ + idx for idx in self.train_indices if idx not in all_val_indices + ] def create_model(self): """ @@ -107,12 +147,12 @@ def create_model(self): def get_train_test_split(self, data): X_train = [data[i] for i in self.train_indices] y_train = [self.labels[i] for i in self.train_indices] - if self.val_indices is None: + if self.test_indices is None: X_test = None y_test = None else: - X_test = [data[i] for i in self.val_indices] - y_test = [self.labels[i] for i in self.val_indices] + X_test = [data[i] for i in self.test_indices] + y_test = [self.labels[i] for i in self.test_indices] return X_train, y_train, X_test, y_test @@ -125,22 +165,25 @@ def run(self, data): """ self._reset_params() model = self.create_model() - skf = KFold(n_splits=self.kfold, shuffle=True, random_state=11) - fold = 0 - X, y, _, _ = self.get_train_test_split(data) + test_X = np.array([data[i] for i in self.test_indices]) + test_y = np.array([self.labels[i] for i in self.test_indices]) + + for fold_idx in range(self.kfold): + fold_train_indices = self.cv_train_indices[fold_idx] + fold_val_indices = self.cv_val_indices[fold_idx] - for train, test in skf.split(X, y): - train_X = np.array(X)[train] - train_y = np.array(y)[train] - test_X = np.array(X)[test] - test_y = np.array(y)[test] - self._run_fold(model, train_X, train_y, test_X, test_y) - fold += 1 + train_X = np.array([data[i] for i in fold_train_indices]) + train_y = np.array([self.labels[i] for i in fold_train_indices]) + val_X = np.array([data[i] for i in fold_val_indices]) + val_y = np.array([self.labels[i] for i in fold_val_indices]) + + self._run_fold(model, train_X, train_y, val_X, val_y, test_X, test_y) return [ self.train_scores.compute_averages(), self.val_scores.compute_averages(), + self.test_scores.compute_averages(), ] def _reset_params(self): @@ -148,48 +191,18 @@ def _reset_params(self): self.training_time = [] self.train_scores = PerformanceMeasure("train", self.performance_measures) self.val_scores = PerformanceMeasure("val", self.performance_measures) + self.test_scores = PerformanceMeasure("test", self.performance_measures) - def _run_fold(self, model, train_X, train_y, test_X, test_y): + def _run_fold(self, model, train_X, train_y, val_X, val_y, test_X, test_y): train_start = time.time() - train_score = model.fit(train_X, train_y, test_X, test_y) + train_score = model.fit(train_X, train_y, val_X, val_y) train_end = time.time() self.training_time.append(train_end - train_start) self.train_scores.add_scores(train_score[0]) + val_score = model.test(val_X, val_y) test_start = time.time() test_score = model.test(np.array(test_X), test_y) test_end = time.time() self.inference_time.append(test_end - test_start) - self.val_scores.add_scores(test_score[0]) - - def create_representation_and_run( - self, - representation: Representation, - modalities: Union[List[Modality], Modality], - ): - self._reset_params() - skf = KFold(n_splits=self.kfold, shuffle=True, random_state=11) - - fold = 0 - X, y, _, _ = self.get_train_test_split(data) - - for train, test in skf.split(X, y): - train_X = np.array(X)[train] - train_y = np.array(y)[train] - test_X = s.transform(np.array(X)[test]) - test_y = np.array(y)[test] - - if isinstance(modalities, Modality): - rep = modality.apply_representation(representation()) - else: - representation().transform( - train_X, train_y - ) # TODO: think about a way how to handle masks - - self._run_fold(train_X, train_y, test_X, test_y) - fold += 1 - - if self.measure_performance: - self.inference_time = np.mean(self.inference_time) - self.training_time = np.mean(self.training_time) - - return [np.mean(train_scores), np.mean(test_scores)] + self.val_scores.add_scores(val_score[0]) + self.test_scores.add_scores(test_score[0]) diff --git a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py index 7735986c2e6..1a348a91df2 100644 --- a/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py +++ b/src/main/python/systemds/scuro/drsearch/unimodal_optimizer.py @@ -54,7 +54,6 @@ def __init__( ): self.modalities = modalities self.tasks = tasks - self.run = None self.save_all_results = save_all_results self.result_path = result_path @@ -65,7 +64,7 @@ def __init__( self.debug = debug self.operator_registry = Registry() - self.operator_performance = UnimodalResults(modalities, tasks, debug, self.run) + self.operator_performance = UnimodalResults(modalities, tasks, debug, True) self._tasks_require_same_dims = True self.expected_dimensions = tasks[0].expected_dim @@ -102,6 +101,17 @@ def store_results(self, file_name=None): with open(file_name, "wb") as f: pickle.dump(self.operator_performance.results, f) + def store_cache(self, file_name=None): + if file_name is None: + import time + + timestr = time.strftime("%Y%m%d-%H%M%S") + file_name = "unimodal_optimizer_cache" + timestr + ".pkl" + + file_name = f"{self.result_path}/{file_name}" + with open(file_name, "wb") as f: + pickle.dump(self.operator_performance.cache, f) + def load_results(self, file_name): with open(file_name, "rb") as f: self.operator_performance.results = pickle.load(f) @@ -390,6 +400,7 @@ def add_result(self, scores, modality, task_name, task_time, combination, dag): entry = ResultEntry( train_score=scores[0].average_scores, val_score=scores[1].average_scores, + test_score=scores[2].average_scores, representation_time=modality.transform_time, task_time=task_time, combination=combination.name if combination else "", @@ -414,22 +425,23 @@ def print_results(self): for entry in self.results[modality][task_name]: print(f"{modality}_{task_name}: {entry}") - def get_k_best_results(self, modality, k, task): + def get_k_best_results(self, modality, k, task, performance_metric_name): """ Get the k best results for the given modality :param modality: modality to get the best results for :param k: number of best results + :param task: task to get the best results for + :param performance_metric_name: name of the performance metric to use for ranking """ task_results = self.results[modality.modality_id][task.model.name] - results = rank_by_tradeoff(task_results)[:k] + results, sorted_indices = rank_by_tradeoff( + task_results, performance_metric_name=performance_metric_name + ) - sorted_indices = sorted( - range(len(task_results)), - key=lambda x: task_results[x].tradeoff_score, - reverse=True, - )[:k] + results = results[:k] + sorted_indices = sorted_indices[:k] task_cache = self.cache.get(modality.modality_id, {}).get(task.model.name, None) if not task_cache: @@ -446,10 +458,11 @@ def get_k_best_results(self, modality, k, task): return results, cache -@dataclass(frozen=True) +@dataclass class ResultEntry: val_score: PerformanceMeasure = None train_score: PerformanceMeasure = None + test_score: PerformanceMeasure = None representation_time: float = 0.0 task_time: float = 0.0 combination: str = "" diff --git a/src/main/python/systemds/scuro/modality/modality.py b/src/main/python/systemds/scuro/modality/modality.py index 98dd631e12c..07f80cbd9ff 100644 --- a/src/main/python/systemds/scuro/modality/modality.py +++ b/src/main/python/systemds/scuro/modality/modality.py @@ -93,6 +93,8 @@ def update_metadata(self): for i, (md_k, md_v) in enumerate(md_copy.items()): updated_md = self.modality_type.update_metadata(md_v, self.data[i]) self.metadata[md_k] = updated_md + if i == 0: + self.data_type = updated_md["data_layout"]["type"] def flatten(self, padding=False): """ diff --git a/src/main/python/systemds/scuro/modality/type.py b/src/main/python/systemds/scuro/modality/type.py index c6f713df240..c2fe38176f7 100644 --- a/src/main/python/systemds/scuro/modality/type.py +++ b/src/main/python/systemds/scuro/modality/type.py @@ -281,7 +281,7 @@ def create_video_metadata(self, frequency, length, width, height, num_channels): md["num_channels"] = num_channels md["timestamp"] = create_timestamps(frequency, length) md["data_layout"]["representation"] = DataLayout.NESTED_LEVEL - md["data_layout"]["type"] = float + md["data_layout"]["type"] = np.float32 md["data_layout"]["shape"] = (width, height, num_channels) return md @@ -291,7 +291,7 @@ def create_image_metadata(self, width, height, num_channels): md["height"] = height md["num_channels"] = num_channels md["data_layout"]["representation"] = DataLayout.SINGLE_LEVEL - md["data_layout"]["type"] = float + md["data_layout"]["type"] = np.float32 md["data_layout"]["shape"] = (width, height, num_channels) return md diff --git a/src/main/python/systemds/scuro/representations/concatenation.py b/src/main/python/systemds/scuro/representations/concatenation.py index bf854a481fd..ea199d58274 100644 --- a/src/main/python/systemds/scuro/representations/concatenation.py +++ b/src/main/python/systemds/scuro/representations/concatenation.py @@ -51,7 +51,7 @@ def execute(self, modalities: List[Modality]): max_emb_size = self.get_max_embedding_size(modalities) size = len(modalities[0].data) - if modalities[0].data.ndim > 2: + if np.array(modalities[0].data).ndim > 2: data = np.zeros((size, max_emb_size, 0)) else: data = np.zeros((size, 0)) diff --git a/src/main/python/systemds/scuro/representations/fusion.py b/src/main/python/systemds/scuro/representations/fusion.py index 8cf67b1cb42..d491dcad6b9 100644 --- a/src/main/python/systemds/scuro/representations/fusion.py +++ b/src/main/python/systemds/scuro/representations/fusion.py @@ -121,29 +121,16 @@ def get_max_embedding_size(self, modalities: List[Modality]): :param modalities: List of modalities :return: maximum embedding size """ - try: - modalities[0].data = np.array(modalities[0].data) - except: - pass - - if isinstance(modalities[0].data[0], list): - max_size = modalities[0].data[0][0].shape[1] - elif isinstance(modalities[0].data, np.ndarray): - max_size = modalities[0].data.shape[1] - else: - max_size = modalities[0].data[0].shape[1] - for idx in range(1, len(modalities)): - if isinstance(modalities[idx].data[0], list): - curr_shape = modalities[idx].data[0][0].shape - elif isinstance(modalities[idx].data, np.ndarray): - curr_shape = modalities[idx].data.shape - else: - curr_shape = modalities[idx].data[0].shape - if len(modalities[idx - 1].data) != len(modalities[idx].data): - raise f"Modality sizes don't match!" - elif len(curr_shape) == 1: - continue - elif curr_shape[1] > max_size: - max_size = curr_shape[1] + max_size = 0 + for m in modalities: + data = m.data + if isinstance(data, memoryview): + data = np.array(data) + arr = np.asarray(data) + if arr.ndim < 2: + continue + emb_size = arr.shape[1] + if emb_size > max_size: + max_size = emb_size return max_size diff --git a/src/main/python/systemds/scuro/representations/timeseries_representations.py b/src/main/python/systemds/scuro/representations/timeseries_representations.py index d1dee67f861..631294809ee 100644 --- a/src/main/python/systemds/scuro/representations/timeseries_representations.py +++ b/src/main/python/systemds/scuro/representations/timeseries_representations.py @@ -46,7 +46,7 @@ def transform(self, modality): feature = self.compute_feature(signal) result.append(feature) - transformed_modality.data = np.vstack(result).astype( + transformed_modality.data = np.vstack(np.array(result)).astype( modality.metadata[list(modality.metadata.keys())[0]]["data_layout"]["type"] ) return transformed_modality diff --git a/src/main/python/tests/scuro/data_generator.py b/src/main/python/tests/scuro/data_generator.py index 3c43cabb3ee..5bec163fe76 100644 --- a/src/main/python/tests/scuro/data_generator.py +++ b/src/main/python/tests/scuro/data_generator.py @@ -198,14 +198,14 @@ def create_visual_modality( ): if max_num_frames > 1: data = [ - np.random.randint( - 0, - 256, + np.random.uniform( + 0.0, + 1.0, (np.random.randint(10, max_num_frames + 1), height, width, 3), - dtype=np.uint8, ) for _ in range(num_instances) ] + metadata = { i: ModalityType.VIDEO.create_metadata( 30, data[i].shape[0], width, height, 3 diff --git a/src/main/python/tests/scuro/test_multimodal_fusion.py b/src/main/python/tests/scuro/test_multimodal_fusion.py index 395a9cd8623..f98824a16a5 100644 --- a/src/main/python/tests/scuro/test_multimodal_fusion.py +++ b/src/main/python/tests/scuro/test_multimodal_fusion.py @@ -174,7 +174,9 @@ def test_multimodal_fusion(self): registry._fusion_operators = [Average, Concatenation, LSTM] unimodal_optimizer = UnimodalOptimizer([audio, text], [task], debug=False) unimodal_optimizer.optimize() - unimodal_optimizer.operator_performance.get_k_best_results(audio, 2, task) + unimodal_optimizer.operator_performance.get_k_best_results( + audio, 2, task, "accuracy" + ) m_o = MultimodalOptimizer( [audio, text], unimodal_optimizer.operator_performance, @@ -187,13 +189,13 @@ def test_multimodal_fusion(self): best_results = sorted( fusion_results[task.model.name], - key=lambda x: getattr(x, "val_score").average_scores["accuracy"], + key=lambda x: getattr(x, "val_score")["accuracy"], reverse=True, )[:2] assert ( - best_results[0].val_score.average_scores["accuracy"] - >= best_results[1].val_score.average_scores["accuracy"] + best_results[0].val_score["accuracy"] + >= best_results[1].val_score["accuracy"] ) def test_parallel_multimodal_fusion(self): @@ -238,7 +240,9 @@ def test_parallel_multimodal_fusion(self): registry._fusion_operators = [Average, Concatenation, LSTM] unimodal_optimizer = UnimodalOptimizer([audio, text], [task], debug=False) unimodal_optimizer.optimize() - unimodal_optimizer.operator_performance.get_k_best_results(audio, 2, task) + unimodal_optimizer.operator_performance.get_k_best_results( + audio, 2, task, "accuracy" + ) m_o = MultimodalOptimizer( [audio, text], unimodal_optimizer.operator_performance, @@ -252,21 +256,21 @@ def test_parallel_multimodal_fusion(self): best_results = sorted( fusion_results[task.model.name], - key=lambda x: getattr(x, "val_score").average_scores["accuracy"], + key=lambda x: getattr(x, "val_score")["accuracy"], reverse=True, ) best_results_parallel = sorted( parallel_fusion_results[task.model.name], - key=lambda x: getattr(x, "val_score").average_scores["accuracy"], + key=lambda x: getattr(x, "val_score")["accuracy"], reverse=True, ) assert len(best_results) == len(best_results_parallel) for i in range(len(best_results)): assert ( - best_results[i].val_score.average_scores["accuracy"] - == best_results_parallel[i].val_score.average_scores["accuracy"] + best_results[i].val_score["accuracy"] + == best_results_parallel[i].val_score["accuracy"] ) diff --git a/src/main/python/tests/scuro/test_unimodal_optimizer.py b/src/main/python/tests/scuro/test_unimodal_optimizer.py index 252dfe997a7..ca54ee64b1d 100644 --- a/src/main/python/tests/scuro/test_unimodal_optimizer.py +++ b/src/main/python/tests/scuro/test_unimodal_optimizer.py @@ -210,7 +210,7 @@ def optimize_unimodal_representation_for_modality(self, modality): registry = Registry() unimodal_optimizer = UnimodalOptimizer([modality], self.tasks, False) - unimodal_optimizer.optimize_parallel() + unimodal_optimizer.optimize() assert ( unimodal_optimizer.operator_performance.modality_ids[0] @@ -218,7 +218,7 @@ def optimize_unimodal_representation_for_modality(self, modality): ) assert len(unimodal_optimizer.operator_performance.task_names) == 2 result, cached = unimodal_optimizer.operator_performance.get_k_best_results( - modality, 1, self.tasks[0] + modality, 1, self.tasks[0], "accuracy" ) assert len(result) == 1 assert len(cached) == 1 diff --git a/src/main/python/tests/scuro/test_unimodal_representations.py b/src/main/python/tests/scuro/test_unimodal_representations.py index 3bc28ee23c5..0313cd29f88 100644 --- a/src/main/python/tests/scuro/test_unimodal_representations.py +++ b/src/main/python/tests/scuro/test_unimodal_representations.py @@ -169,7 +169,6 @@ def test_image_representations(self): def test_video_representations(self): video_representations = [ CLIPVisual(), - ColorHistogram(), I3D(), X3D(), VGG19(),