Merge branch 'impswp' into g2048-65k

Author: kywch

pufferlib/config/default.ini

@@ -73,85 +73,73 @@ prune_pareto = True
 #distribution = uniform_pow2
 #min = 1
 #max = 16
-#mean = 8
 #scale = auto
 
 # TODO: Elim from base
 [sweep.train.total_timesteps]
 distribution = log_normal
 min = 3e7
 max = 1e10
-mean = 2e8
 scale = time
 
 [sweep.train.bptt_horizon]
 distribution = uniform_pow2
 min = 16
 max = 64
-mean = 64
 scale = auto
 
 [sweep.train.minibatch_size]
 distribution = uniform_pow2
 min = 8192
 max = 65536
-mean = 32768
 scale = auto
 
 [sweep.train.learning_rate]
 distribution = log_normal
 min = 0.00001
-mean = 0.01
 max = 0.1
 scale = 0.5
 
 [sweep.train.ent_coef]
 distribution = log_normal
 min = 0.00001
-mean = 0.01
 max = 0.2
 scale = auto
 
 [sweep.train.gamma]
 distribution = logit_normal
 min = 0.8
-mean = 0.98
 max = 0.9999
 scale = auto
 
 [sweep.train.gae_lambda]
 distribution = logit_normal
 min = 0.6
-mean = 0.95
 max = 0.995
 scale = auto
 
 [sweep.train.vtrace_rho_clip]
 distribution = uniform
 min = 0.1
 max = 5.0
-mean = 1.0
 scale = auto
 
 [sweep.train.vtrace_c_clip]
 distribution = uniform
 min = 0.1
 max = 5.0
-mean = 1.0
 scale = auto
 
 #[sweep.train.update_epochs]
 #distribution = int_uniform
 #min = 1
 #max = 8
-#mean = 1
 #scale = 2.0
 
 [sweep.train.clip_coef]
 distribution = uniform
 min = 0.01
 max = 1.0
-mean = 0.2
 scale = auto
 
 # Optimal vf clip can be lower than 0.1,
@@ -160,54 +148,46 @@ scale = auto
 distribution = uniform
 min = 0.1
 max = 5.0
-mean = 0.2
 scale = auto
 
 [sweep.train.vf_coef]
 distribution = uniform
 min = 0.1
 max = 5.0
-mean = 2.0
 scale = auto
 
 [sweep.train.max_grad_norm]
 distribution = uniform
 min = 0.1
-mean = 1.0
 max = 5.0
 scale = auto
 
 [sweep.train.adam_beta1]
 distribution = logit_normal
 min = 0.5
-mean = 0.9
 max = 0.999
 scale = auto
 
 [sweep.train.adam_beta2]
 distribution = logit_normal
 min = 0.9
-mean = 0.999
 max = 0.99999
 scale = auto
 
 [sweep.train.adam_eps]
 distribution = log_normal
 min = 1e-14
-mean = 1e-8
 max = 1e-4
 scale = auto
 
 [sweep.train.prio_alpha]
 distribution = logit_normal
 min = 0.1
-mean = 0.85
 max = 0.99
 scale = auto
 
 [sweep.train.prio_beta0]
 distribution = logit_normal
 min = 0.1
-mean = 0.85
 max = 0.99
 scale = auto

pufferlib/sweep.py

@@ -31,22 +31,22 @@ def default_tensor_dtype(dtype):
         torch.set_default_dtype(old_dtype)
 
 class Space:
-    def __init__(self, min, max, scale, mean, is_integer=False):
+    def __init__(self, min, max, scale, is_integer=False):
         self.min = min
         self.max = max
         self.scale = scale
-        self.mean = mean # TODO: awkward to have just this normalized
         self.norm_min = self.normalize(min)
         self.norm_max = self.normalize(max)
-        self.norm_mean = self.normalize(mean)
+        # Since min/max are normalized from -1 to 1, just use 0 as a mean
+        self.norm_mean = 0
         self.is_integer = is_integer
 
 class Linear(Space):
-    def __init__(self, min, max, scale, mean, is_integer=False):
+    def __init__(self, min, max, scale, is_integer=False):
         if scale == 'auto':
             scale = 0.5
 
-        super().__init__(min, max, scale, mean, is_integer)
+        super().__init__(min, max, scale, is_integer)
 
     def normalize(self, value):
         #assert isinstance(value, (int, float))
@@ -61,12 +61,12 @@ def unnormalize(self, value):
         return value
 
 class Pow2(Space):
-    def __init__(self, min, max, scale, mean, is_integer=False):
+    def __init__(self, min, max, scale, is_integer=False):
         if scale == 'auto':
             scale = 0.5
             #scale = 2 / (np.log2(max) - np.log2(min))
 
-        super().__init__(min, max, scale, mean, is_integer)
+        super().__init__(min, max, scale, is_integer)
 
     def normalize(self, value):
         #assert isinstance(value, (int, float))
@@ -83,14 +83,14 @@ def unnormalize(self, value):
 class Log(Space):
     base: int = 10
 
-    def __init__(self, min, max, scale, mean, is_integer=False):
+    def __init__(self, min, max, scale, is_integer=False):
         if scale == 'time':
             # TODO: Set scaling param intuitively based on number of jumps from min to max
             scale = 1 / (np.log2(max) - np.log2(min))
         elif scale == 'auto':
             scale = 0.5
 
-        super().__init__(min, max, scale, mean, is_integer)
+        super().__init__(min, max, scale, is_integer)
 
     def normalize(self, value):
         #assert isinstance(value, (int, float))
@@ -109,11 +109,11 @@ def unnormalize(self, value):
 class Logit(Space):
     base: int = 10
 
-    def __init__(self, min, max, scale, mean, is_integer=False):
+    def __init__(self, min, max, scale, is_integer=False):
         if scale == 'auto':
             scale = 0.5
 
-        super().__init__(min, max, scale, mean, is_integer)
+        super().__init__(min, max, scale, is_integer)
 
     def normalize(self, value):
         #assert isinstance(value, (int, float))
@@ -147,12 +147,10 @@ def _params_from_puffer_sweep(sweep_config, only_include=None):
 
         assert 'distribution' in param
         distribution = param['distribution']
-        search_center = param['mean']
         kwargs = dict(
             min=param['min'],
             max=param['max'],
             scale=param['scale'],
-            mean=search_center,
         )
         if distribution == 'uniform':
             space = Linear(**kwargs)
@@ -432,7 +430,6 @@ def __init__(self,
             num_random_samples = 10,
             global_search_scale = 1,
             suggestions_per_pareto = 256,
-            seed_with_search_center = True,
             expansion_rate = 0.25,
             gp_training_iter = 50,
             gp_learning_rate = 0.001,
@@ -452,7 +449,6 @@ def __init__(self,
         self.hyperparameters = Hyperparameters(sweep_config)
         self.global_search_scale = global_search_scale
         self.suggestions_per_pareto = suggestions_per_pareto
-        self.seed_with_search_center = seed_with_search_center
         self.resample_frequency = resample_frequency
         self.max_suggestion_cost = _max_suggestion_cost
         self.expansion_rate = expansion_rate
@@ -641,8 +637,8 @@ def suggest(self, fill):
 
         ### Sample suggestions
         search_centers = np.stack([e['input'] for e in candidates])
-        num_sample = len(candidates) * self.suggestions_per_pareto
-        suggestions = self.hyperparameters.sample(num_sample, mu=search_centers)
+        suggestions = self.hyperparameters.sample(
+            len(candidates)*self.suggestions_per_pareto, mu=search_centers)
 
         dedup_indices = self._filter_near_duplicates(suggestions)
         suggestions = suggestions[dedup_indices]