iALS tuning strategy of doubling dimensions (#95)

* - Implement doubling dimension strategy.
- Use `Sequence[Suggestion]` instead of `List[Suggestion]` for covariance.

* Add test for doubling dimension tuning.

* - Add example of doubling dimension strategy.
- increment `n_trials_following` default value.

* Added docstrings

Author: tohtsky

examples/movielens/movielens_20m_cold.py

@@ -4,8 +4,8 @@
 import pandas as pd
 from scipy import sparse as sps
 
+from irspack import EvaluatorWithColdUser, IALSRecommender
 from irspack.dataset.movielens import MovieLens20MDataManager
-from irspack.evaluator import EvaluatorWithColdUser
 from irspack.optimizers import (
     AsymmetricCosineKNNOptimizer,
     BaseOptimizer,
@@ -69,7 +69,6 @@
         (AsymmetricCosineKNNOptimizer, 40, dict()),
         (P3alphaOptimizer, 30, dict(alpha=1)),
         (RP3betaOptimizer, 40, dict(alpha=1)),
-        (IALSOptimizer, 40, dict()),
         (DenseSLIMOptimizer, 20, dict()),
         (
             MultVAEOptimizer,
@@ -87,7 +86,9 @@
             valid_evaluator,
             fixed_params=config,
         )
-        (best_param, validation_result_df) = optimizer.optimize(n_trials=n_trials)
+        (best_param, validation_result_df) = optimizer.optimize(
+            n_trials=n_trials, random_seed=0
+        )
         validation_result_df["recommender_name"] = recommender_name
         validation_results.append(validation_result_df)
         pd.concat(validation_results).to_csv(f"validation_scores.csv")
@@ -100,3 +101,33 @@
         )
         with open("test_results.json", "w") as ofs:
             json.dump(test_results, ofs, indent=2)
+
+    # Tuning following the strategy of
+    # "Revisiting the Performance of iALS on Item Recommendation Benchmarks"
+    # https://arxiv.org/abs/2110.14037
+    ials_optimizer = IALSOptimizer(
+        data_train.X_all,
+        valid_evaluator,
+    )
+    (
+        best_param_ials,
+        validation_result_df_ials,
+    ) = ials_optimizer.optimize_doubling_dimension(
+        initial_dimension=128,
+        maximal_dimension=1024,
+        random_seed=0,
+        n_trials_initial=80,
+        n_trials_following=40,
+    )
+    validation_result_df_ials["recommender_name"] = "IALSRecommender"
+    validation_results.append(validation_result_df_ials)
+    pd.concat(validation_results).to_csv(f"validation_scores.csv")
+    test_recommender_ials = IALSRecommender(X_train_val_all, **best_param_ials)
+    test_recommender_ials.learn()
+    test_scores_ials = test_evaluator.get_scores(test_recommender_ials, [20, 50, 100])
+
+    test_results.append(
+        dict(name="IALSRecommender", best_param=best_param_ials, **test_scores_ials)
+    )
+    with open("test_results.json", "w") as ofs:
+        json.dump(test_results, ofs, indent=2)

irspack/optimizers/__init__.py

@@ -7,7 +7,6 @@
     CosineUserKNNOptimizer,
     DenseSLIMOptimizer,
     EDLAEOptimizer,
-    IALSOptimizer,
     JaccardKNNOptimizer,
     P3alphaOptimizer,
     RP3betaOptimizer,
@@ -17,6 +16,7 @@
 )
 from irspack.optimizers.autopilot import autopilot
 from irspack.optimizers.base_optimizer import get_optimizer_class
+from irspack.optimizers.ials import IALSOptimizer
 
 __all__ = [
     "BaseOptimizer",

irspack/optimizers/_optimizers.py

@@ -1,9 +1,7 @@
-import logging
-from typing import Any, Dict, List, Optional
+from typing import Sequence
 
 from irspack.definitions import InteractionMatrix
 
-from ..evaluator import Evaluator
 from ..optimizers.base_optimizer import (
     BaseOptimizer,
     BaseOptimizerWithEarlyStopping,
@@ -23,7 +21,6 @@
     CosineUserKNNRecommender,
     DenseSLIMRecommender,
     EDLAERecommender,
-    IALSRecommender,
     JaccardKNNRecommender,
     P3alphaRecommender,
     RP3betaRecommender,
@@ -51,64 +48,24 @@ def _get_maximal_n_components_for_budget(
 
 
 class TopPopOptimizer(BaseOptimizer):
-    default_tune_range: List[Suggestion] = []
+    default_tune_range: Sequence[Suggestion] = []
     recommender_class = TopPopRecommender
 
     @classmethod
     def tune_range_given_memory_budget(
         cls, X: InteractionMatrix, memory_budget: int
-    ) -> List[Suggestion]:
+    ) -> Sequence[Suggestion]:
         return []
 
 
-class IALSOptimizer(BaseOptimizerWithEarlyStopping):
-    default_tune_range = [
-        IntegerSuggestion("n_components", 4, 300),
-        LogUniformSuggestion("alpha0", 3e-3, 1),
-        LogUniformSuggestion("reg", 1e-4, 1e-1),
-    ]
-    recommender_class = IALSRecommender
-
-    def __init__(
-        self,
-        data: InteractionMatrix,
-        val_evaluator: Evaluator,
-        logger: Optional[logging.Logger] = None,
-        suggest_overwrite: List[Suggestion] = [],
-        fixed_params: Dict[str, Any] = {},
-        max_epoch: int = 16,
-        validate_epoch: int = 1,
-        score_degradation_max: int = 5,
-    ):
-        super().__init__(
-            data,
-            val_evaluator,
-            logger=logger,
-            suggest_overwrite=suggest_overwrite,
-            fixed_params=fixed_params,
-            max_epoch=max_epoch,
-            validate_epoch=validate_epoch,
-            score_degradation_max=score_degradation_max,
-        )
-
-    @classmethod
-    def tune_range_given_memory_budget(
-        cls, X: InteractionMatrix, memory_budget: int
-    ) -> List[Suggestion]:
-        n_components = _get_maximal_n_components_for_budget(X, memory_budget, 300)
-        return [
-            IntegerSuggestion("n_components", 4, n_components),
-        ]
-
-
 class DenseSLIMOptimizer(BaseOptimizer):
-    default_tune_range: List[Suggestion] = [LogUniformSuggestion("reg", 1, 1e4)]
+    default_tune_range: Sequence[Suggestion] = [LogUniformSuggestion("reg", 1, 1e4)]
     recommender_class = DenseSLIMRecommender
 
     @classmethod
     def tune_range_given_memory_budget(
         cls, X: InteractionMatrix, memory_budget: int
-    ) -> List[Suggestion]:
+    ) -> Sequence[Suggestion]:
         n_items: int = X.shape[1]
         if (1e6 * memory_budget) < (4 * 2 * n_items**2):
             raise LowMemoryError(
@@ -118,7 +75,7 @@ def tune_range_given_memory_budget(
 
 
 class EDLAEOptimizer(BaseOptimizer):
-    default_tune_range: List[Suggestion] = [
+    default_tune_range: Sequence[Suggestion] = [
         LogUniformSuggestion("reg", 1, 1e4),
         UniformSuggestion("dropout_p", 0.0, 0.99),
     ]
@@ -127,7 +84,7 @@ class EDLAEOptimizer(BaseOptimizer):
     @classmethod
     def tune_range_given_memory_budget(
         cls, X: InteractionMatrix, memory_budget: int
-    ) -> List[Suggestion]:
+    ) -> Sequence[Suggestion]:
         n_items: int = X.shape[1]
         if (1e6 * memory_budget) < (4 * 2 * n_items**2):
             raise LowMemoryError(
@@ -146,7 +103,7 @@ class TruncatedSVDOptimizer(BaseOptimizer):
         @classmethod
         def tune_range_given_memory_budget(
             cls, X: InteractionMatrix, memory_budget: int
-        ) -> List[Suggestion]:
+        ) -> Sequence[Suggestion]:
             n_components = _get_maximal_n_components_for_budget(X, memory_budget, 512)
             return [
                 IntegerSuggestion("n_components", 4, n_components),
@@ -163,7 +120,7 @@ class NMFOptimizer(BaseOptimizer):
         @classmethod
         def tune_range_given_memory_budget(
             cls, X: InteractionMatrix, memory_budget: int
-        ) -> List[Suggestion]:
+        ) -> Sequence[Suggestion]:
             n_components = _get_maximal_n_components_for_budget(X, memory_budget, 512)
             return [
                 IntegerSuggestion("n_components", 4, n_components),
@@ -177,7 +134,7 @@ class SimilarityBasedOptimizerBase(BaseOptimizer):
     @classmethod
     def tune_range_given_memory_budget(
         cls, X: InteractionMatrix, memory_budget: int
-    ) -> List[Suggestion]:
+    ) -> Sequence[Suggestion]:
         top_k_max = min(int(1e6 * memory_budget / 4 // (X.shape[1] + 1)), 1024)
         if top_k_max <= 4:
             raise LowMemoryError(
@@ -249,7 +206,7 @@ class UserSimilarityBasedOptimizerBase(BaseOptimizer):
     @classmethod
     def tune_range_given_memory_budget(
         cls, X: InteractionMatrix, memory_budget: int
-    ) -> List[Suggestion]:
+    ) -> Sequence[Suggestion]:
         top_k_max = min(int(1e6 * memory_budget / 4 // (X.shape[0] + 1)), 1024)
         return [
             IntegerSuggestion("top_k", 4, top_k_max),
@@ -287,7 +244,7 @@ class BPRFMOptimizer(BaseOptimizerWithEarlyStopping):
         @classmethod
         def tune_range_given_memory_budget(
             cls, X: InteractionMatrix, memory_budget: int
-        ) -> List[Suggestion]:
+        ) -> Sequence[Suggestion]:
             # memory usage will be roughly 4 (float) * (n_users + n_items) * k
             n_components = _get_maximal_n_components_for_budget(X, memory_budget, 300)
             return [
@@ -312,7 +269,7 @@ class MultVAEOptimizer(BaseOptimizerWithEarlyStopping):
         @classmethod
         def tune_range_given_memory_budget(
             cls, X: InteractionMatrix, memory_budget: int
-        ) -> List[Suggestion]:
+        ) -> Sequence[Suggestion]:
             if memory_budget * 1e6 > (X.shape[1] * 2048 * 8):
                 raise LowMemoryError(
                     f"Memory budget {memory_budget} too small for MultVAE to work."

irspack/optimizers/autopilot.py

@@ -6,7 +6,7 @@
 from multiprocessing import Pipe as mp_pipe
 from multiprocessing import Process
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional, Tuple, Type
+from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Type
 from uuid import uuid1
 
 import numpy as np
@@ -37,7 +37,7 @@ def search_one(
     X: InteractionMatrix,
     evaluator: Evaluator,
     optimizer_names: List[str],
-    suggest_overwrites: Dict[str, List[Suggestion]],
+    suggest_overwrites: Dict[str, Sequence[Suggestion]],
     db_url: str,
     study_name: str,
     random_seed: int,
@@ -72,7 +72,7 @@ def __init__(
         X: InteractionMatrix,
         evaluator: Evaluator,
         optimizer_names: List[str],
-        suggest_overwrites: Dict[str, List[Suggestion]],
+        suggest_overwrites: Dict[str, Sequence[Suggestion]],
         db_url: str,
         study_name: str,
         random_seed: int,
@@ -111,7 +111,7 @@ def __init__(
         X: InteractionMatrix,
         evaluator: Evaluator,
         optimizer_names: List[str],
-        suggest_overwrites: Dict[str, List[Suggestion]],
+        suggest_overwrites: Dict[str, Sequence[Suggestion]],
         db_url: str,
         study_name: str,
         random_seed: int,
@@ -156,7 +156,7 @@ def __init__(
         X: InteractionMatrix,
         evaluator: Evaluator,
         optimizer_names: List[str],
-        suggest_overwrites: Dict[str, List[Suggestion]],
+        suggest_overwrites: Dict[str, Sequence[Suggestion]],
         db_url: str,
         study_name: str,
         random_seed: int,
@@ -275,7 +275,7 @@ def autopilot(
     if storage is not None and study_name is None:
         raise ValueError('"study_name" must be specified if "storage" is given.')
     RNS = np.random.RandomState(random_seed)
-    suggest_overwrites: Dict[str, List[Suggestion]] = {}
+    suggest_overwrites: Dict[str, Sequence[Suggestion]] = {}
     optimizer_names: List[str] = []
     for rec_name in algorithms:
         optimizer_class_name = rec_name + "Optimizer"