Add UpdateLambda, different optimizer for encoder

Author: antoinedemathelin

adapt/base.py

@@ -889,7 +889,7 @@ def fit(self, X, y=None, Xt=None, yt=None, domains=None, **fit_params):
         Returns
         -------
         self : returns an instance of self
-        """        
+        """
         set_random_seed(self.random_state)
 
         # 1. Initialize networks
@@ -906,7 +906,6 @@ def fit(self, X, y=None, Xt=None, yt=None, domains=None, **fit_params):
                     shape = first_elem[0].shape
             else:
                 shape = X.shape[1:]
-            print(shape)
             self._initialize_weights(shape)
 
         # 2. Get Fit params
@@ -984,9 +983,9 @@ def fit(self, X, y=None, Xt=None, yt=None, domains=None, **fit_params):
         self._save_validation_data(X, Xt)
 
         # 4. Get validation data
-        validation_data = self._check_validation_data(validation_data,
-                                                      validation_batch_size,
-                                                      shuffle)
+#         validation_data = self._check_validation_data(validation_data,
+#                                                       validation_batch_size,
+#                                                       shuffle)
 
         if validation_data is None and validation_split>0.:
             if shuffle:
@@ -1053,20 +1052,20 @@ def fit(self, X, y=None, Xt=None, yt=None, domains=None, **fit_params):
 
             self._initialize_pretain_networks()
 
-        # 6. Training
+        # 6. Compile
         if (not self._is_compiled) or (self.pretrain_):
             self.compile()
 
         if not hasattr(self, "history_"):
             self.history_ = {}
 
+        # .7 Training
         if shuffle:
             dataset = tf.data.Dataset.zip((dataset_src, dataset_tgt)).shuffle(buffer_size=1024).batch(batch_size)
         else:
             dataset = tf.data.Dataset.zip((dataset_src, dataset_tgt)).batch(batch_size)
-             
+
         self.pretrain_ = False
-        self.steps_ = tf.Variable(0.)
 
         hist = super().fit(dataset, validation_data=validation_data, **fit_params)
 
@@ -1247,6 +1246,12 @@ def compile(self,
         super().compile(
             **compile_params
         )
+        
+        # Set optimizer for encoder and discriminator
+        if not hasattr(self, "optimizer_enc"):
+            self.optimizer_enc = self.optimizer
+        if not hasattr(self, "optimizer_disc"):
+            self.optimizer_disc = self.optimizer
 
 
     def call(self, inputs):
@@ -1431,7 +1436,7 @@ def score_estimator(self, X, y, sample_weight=None):
         score : float
             Score.
         """
-        if np.prod(X.shape) <= 10**8:
+        if hasattr(X, "shape") and np.prod(X.shape) <= 10**8:
             score = self.evaluate(
                     X, y,
                     sample_weight=sample_weight,
@@ -1447,20 +1452,20 @@ def score_estimator(self, X, y, sample_weight=None):
         return score
 
 
-    def _check_validation_data(self, validation_data, batch_size, shuffle):
-        if isinstance(validation_data, tuple):
-            X_val = validation_data[0]
-            y_val = validation_data[1]
-        
-            validation_data = tf.data.Dataset.zip(
-                (tf.data.Dataset.from_tensor_slices(X_val),
-                 tf.data.Dataset.from_tensor_slices(y_val))
-            )
-            if shuffle:
-                validation_data = validation_data.shuffle(buffer_size=1024).batch(batch_size)
-            else:
-                validation_data = validation_data.batch(batch_size)
-        return validation_data
+#     def _check_validation_data(self, validation_data, batch_size, shuffle):
+#         if isinstance(validation_data, tuple):
+#             X_val = validation_data[0]
+#             y_val = validation_data[1]
+        
+#             validation_data = tf.data.Dataset.zip(
+#                 (tf.data.Dataset.from_tensor_slices(X_val),
+#                  tf.data.Dataset.from_tensor_slices(y_val))
+#             )
+#             if shuffle:
+#                 validation_data = validation_data.shuffle(buffer_size=1024).batch(batch_size)
+#             else:
+#                 validation_data = validation_data.batch(batch_size)
+#         return validation_data
 
 
     def _get_legal_params(self, params):
@@ -1476,7 +1481,7 @@ def _get_legal_params(self, params):
         if (optimizer is not None) and (not isinstance(optimizer, str)):
             legal_params_fct.append(optimizer.__init__)
 
-        legal_params = ["domain", "val_sample_size"]
+        legal_params = ["domain", "val_sample_size", "optimizer_enc", "optimizer_disc"]
         for func in legal_params_fct:
             args = [
                 p.name

adapt/feature_based/_adda.py

@@ -169,7 +169,7 @@ def pretrain_step(self, data):
         Xs, Xt, ys, yt = self._unpack_data(data)
 
         # loss
-        with tf.GradientTape() as tape:                       
+        with tf.GradientTape() as task_tape, tf.GradientTape() as enc_tape:                       
             # Forward pass
             Xs_enc = self.encoder_src_(Xs, training=True)
             ys_pred = self.task_(Xs_enc, training=True)
@@ -179,14 +179,19 @@ def pretrain_step(self, data):
 
             # Compute the loss value
             loss = self.task_loss_(ys, ys_pred)
-            loss += sum(self.task_.losses) + sum(self.encoder_src_.losses)
+            task_loss = loss + sum(self.task_.losses)
+            enc_loss = loss + sum(self.encoder_src_.losses)
 
         # Compute gradients
-        trainable_vars = self.task_.trainable_variables + self.encoder_src_.trainable_variables
-        gradients = tape.gradient(loss, trainable_vars)
+        trainable_vars_task = self.task_.trainable_variables
+        trainable_vars_enc = self.encoder_src_.trainable_variables
+
+        gradients_task = task_tape.gradient(task_loss, trainable_vars_task)
+        gradients_enc = enc_tape.gradient(enc_loss, trainable_vars_enc)
 
         # Update weights
-        self.optimizer.apply_gradients(zip(gradients, trainable_vars))
+        self.optimizer.apply_gradients(zip(gradients_task, trainable_vars_task))
+        self.optimizer_enc.apply_gradients(zip(gradients_enc, trainable_vars_enc))
 
         # Update metrics
         self.compiled_metrics.update_state(ys, ys_pred)
@@ -206,55 +211,48 @@ def train_step(self, data):
             Xs, Xt, ys, yt = self._unpack_data(data)
 
             # loss
-            with tf.GradientTape() as task_tape, tf.GradientTape() as enc_tape, tf.GradientTape() as disc_tape:                       
+            with tf.GradientTape() as enc_tape, tf.GradientTape() as disc_tape:                       
                 # Forward pass
-                Xs_enc = self.encoder_src_(Xs, training=True)
-                ys_pred = self.task_(Xs_enc, training=True)
+                if self.pretrain:
+                    Xs_enc = self.encoder_src_(Xs, training=False)
+                else:
+                    # encoder src is not needed if pretrain=False
+                    Xs_enc = Xs
+                    
                 ys_disc = self.discriminator_(Xs_enc, training=True)
 
                 Xt_enc = self.encoder_(Xt, training=True)
                 yt_disc = self.discriminator_(Xt_enc, training=True)
 
-                # Reshape
-                ys_pred = tf.reshape(ys_pred, tf.shape(ys))
-
                 # Compute the loss value
-                task_loss = self.task_loss_(ys, ys_pred)
-
                 disc_loss = (-tf.math.log(ys_disc + EPS)
                              -tf.math.log(1-yt_disc + EPS))
 
                 enc_loss = -tf.math.log(yt_disc + EPS)
 
-                task_loss = tf.reduce_mean(task_loss)
                 disc_loss = tf.reduce_mean(disc_loss)
                 enc_loss = tf.reduce_mean(enc_loss)
 
-                task_loss += sum(self.task_.losses)
                 disc_loss += sum(self.discriminator_.losses)
                 enc_loss += sum(self.encoder_.losses)
 
             # Compute gradients
-            trainable_vars_task = self.task_.trainable_variables
             trainable_vars_enc = self.encoder_.trainable_variables
             trainable_vars_disc = self.discriminator_.trainable_variables
 
-            gradients_task = task_tape.gradient(task_loss, trainable_vars_task)
             gradients_enc = enc_tape.gradient(enc_loss, trainable_vars_enc)
             gradients_disc = disc_tape.gradient(disc_loss, trainable_vars_disc)
 
             # Update weights
-            self.optimizer.apply_gradients(zip(gradients_task, trainable_vars_task))
-            self.optimizer.apply_gradients(zip(gradients_enc, trainable_vars_enc))
-            self.optimizer.apply_gradients(zip(gradients_disc, trainable_vars_disc))
+            self.optimizer_enc.apply_gradients(zip(gradients_enc, trainable_vars_enc))
+            self.optimizer_disc.apply_gradients(zip(gradients_disc, trainable_vars_disc))
 
             # Update metrics
-            self.compiled_metrics.update_state(ys, ys_pred)
-            self.compiled_loss(ys, ys_pred)
+            # self.compiled_metrics.update_state(ys, ys_pred)
+            # self.compiled_loss(ys, ys_pred)
             # Return a dict mapping metric names to current value
-            logs = {m.name: m.result() for m in self.metrics}
-            disc_metrics = self._get_disc_metrics(ys_disc, yt_disc)
-            logs.update(disc_metrics)
+            # logs = {m.name: m.result() for m in self.metrics}
+            logs = self._get_disc_metrics(ys_disc, yt_disc)
             return logs
 
 
@@ -275,12 +273,14 @@ def _get_disc_metrics(self, ys_disc, yt_disc):
     def _initialize_weights(self, shape_X):
         # Init weights encoder
         self(np.zeros((1,) + shape_X))
-        self.encoder_(np.zeros((1,) + shape_X))
 
         # Set same weights to encoder_src
-        self.encoder_src_ = check_network(self.encoder_,
-                                          copy=True,
-                                          name="encoder_src")
+        if self.pretrain:
+            # encoder src is not needed if pretrain=False
+            self.encoder_(np.zeros((1,) + shape_X))
+            self.encoder_src_ = check_network(self.encoder_,
+                                              copy=True,
+                                              name="encoder_src")
 
 
     def transform(self, X, domain="tgt"):

adapt/feature_based/_cdan.py

@@ -255,8 +255,8 @@ def train_step(self, data):
 
         # Update weights
         self.optimizer.apply_gradients(zip(gradients_task, trainable_vars_task))
-        self.optimizer.apply_gradients(zip(gradients_enc, trainable_vars_enc))
-        self.optimizer.apply_gradients(zip(gradients_disc, trainable_vars_disc))
+        self.optimizer_enc.apply_gradients(zip(gradients_enc, trainable_vars_enc))
+        self.optimizer_disc.apply_gradients(zip(gradients_disc, trainable_vars_disc))
 
         # Update metrics
         self.compiled_metrics.update_state(ys, ys_pred)

adapt/feature_based/_dann.py

@@ -58,12 +58,8 @@ class DANN(BaseAdaptDeep):
     
     Parameters
     ----------        
-    lambda_ : float or None (default=0.1)
+    lambda_ : float (default=0.1)
         Trade-off parameter.
-        If ``None``, ``lambda_`` increases gradually
-        according to the following formula:
-        ``lambda_`` = 2/(1 + exp(-``gamma`` * p)) - 1.
-        With p growing from 0 to 1 during training.
         
     gamma : float (default=10.0)
         Increase rate parameter.
@@ -137,24 +133,9 @@ def __init__(self,
     def train_step(self, data):
         # Unpack the data.
         Xs, Xt, ys, yt = self._unpack_data(data)
-        
-        if self.lambda_ is None:
-            _is_lambda_None = 1.
-            lambda_ = 0.
-        else:
-            _is_lambda_None = 0.
-            lambda_ = float(self.lambda_)
 
         # loss
-        with tf.GradientTape() as task_tape, tf.GradientTape() as enc_tape, tf.GradientTape() as disc_tape:           
-            
-            # Compute lambda
-            self.steps_.assign_add(1.)
-            progress = self.steps_ / self.total_steps_
-            _lambda_ = 2. / (1. + tf.exp(-self.gamma * progress)) - 1.
-            _lambda_ = (_is_lambda_None * _lambda_ +
-                        (1. - _is_lambda_None) * lambda_)
-            
+        with tf.GradientTape() as task_tape, tf.GradientTape() as enc_tape, tf.GradientTape() as disc_tape:
             # Forward pass
             Xs_enc = self.encoder_(Xs, training=True)
             ys_pred = self.task_(Xs_enc, training=True)
@@ -175,7 +156,7 @@ def train_step(self, data):
             task_loss = tf.reduce_mean(task_loss)
             disc_loss = tf.reduce_mean(disc_loss)
 
-            enc_loss = task_loss - _lambda_ * disc_loss
+            enc_loss = task_loss - self.lambda_ * disc_loss
 
             task_loss += sum(self.task_.losses)
             disc_loss += sum(self.discriminator_.losses)
@@ -193,8 +174,8 @@ def train_step(self, data):
 
         # Update weights
         self.optimizer.apply_gradients(zip(gradients_task, trainable_vars_task))
-        self.optimizer.apply_gradients(zip(gradients_enc, trainable_vars_enc))
-        self.optimizer.apply_gradients(zip(gradients_disc, trainable_vars_disc))
+        self.optimizer_enc.apply_gradients(zip(gradients_enc, trainable_vars_enc))
+        self.optimizer_disc.apply_gradients(zip(gradients_disc, trainable_vars_disc))
 
         # Update metrics
         self.compiled_metrics.update_state(ys, ys_pred)

adapt/feature_based/_deepcoral.py

@@ -140,7 +140,7 @@ def train_step(self, data):
             _match_mean = 0.
 
         # loss
-        with tf.GradientTape() as tape:           
+        with tf.GradientTape() as task_tape, tf.GradientTape() as enc_tape:           
 
             # Forward pass
             Xs_enc = self.encoder_(Xs, training=True)
@@ -187,17 +187,19 @@ def train_step(self, data):
             disc_loss_mean = tf.reduce_mean(disc_loss_mean)
             disc_loss = self.lambda_ * (disc_loss_cov + _match_mean * disc_loss_mean)
 
-            loss = task_loss + disc_loss
-            
-            loss += sum(self.task_.losses) + sum(self.encoder_.losses)
+            task_loss += sum(self.task_.losses)
+            disc_loss += sum(self.encoder_.losses)
 
         # Compute gradients
-        trainable_vars = self.task_.trainable_variables + self.encoder_.trainable_variables
+        trainable_vars_task = self.task_.trainable_variables
+        trainable_vars_enc = self.encoder_.trainable_variables
 
-        gradients = tape.gradient(loss, trainable_vars)
+        gradients_task = task_tape.gradient(task_loss, trainable_vars_task)
+        gradients_enc = enc_tape.gradient(disc_loss, trainable_vars_enc)
 
         # Update weights
-        self.optimizer.apply_gradients(zip(gradients, trainable_vars))
+        self.optimizer.apply_gradients(zip(gradients_task, trainable_vars_task))
+        self.optimizer_enc.apply_gradients(zip(gradients_enc, trainable_vars_enc))
 
         # Update metrics
         self.compiled_metrics.update_state(ys, ys_pred)