add timing of backwards pass (#39)

Author: pat-schmitt

agile1d/core/cost_function.py

@@ -324,8 +324,10 @@ def cost_fct(unknown_parameters, data_logger):
     c = c_terms.mean() + cost_lambda * reg_terms.mean()
 
     log.debug('gradient calculation')
+    start_time_gradient_calc = time.time()
     # calculate the gradient for the control variables
     c.backward()
+    gradient_calc_time = time.time() - start_time_gradient_calc
 
     log.debug('get gradients')
     # get gradient/s as numpy array
@@ -351,6 +353,8 @@ def cost_fct(unknown_parameters, data_logger):
                                         unknown_parameters_descaled)
     data_logger.save_data_in_datalogger('time_needed',
                                         time.time() - data_logger.start_time)
+    data_logger.save_data_in_datalogger('gradient_time_needed',
+                                        gradient_calc_time)
 
     data_logger.fct_calls = np.append(data_logger.fct_calls,
                                       data_logger.fct_calls[-1] + 1)

agile1d/core/data_logging.py

@@ -138,6 +138,7 @@ def __init__(self, gdir, fls_init, inversion_input, climate_filename,
         self.reg_terms = None
         self.c_terms_description = None
         self.time_needed = None
+        self.gradient_time_needed = None
         self.grads = None
         self.flowlines = None
         self.initial_flux = None
@@ -248,6 +249,7 @@ def filter_data_from_optimization(self):
         self.c_terms = self.c_terms[index]
         self.reg_terms = self.reg_terms[index]
         self.time_needed = self.squeeze_generic(self.time_needed[index])
+        self.gradient_time_needed = self.squeeze_generic(self.gradient_time_needed[index])
         self.flowlines = self.squeeze_generic(self.flowlines[index])
         self.initial_flux = self.squeeze_generic(self.initial_flux[index])
         self.sfc_h_start = self.squeeze_generic(self.sfc_h_start[index])
@@ -279,6 +281,7 @@ def create_and_save_dataset(self):
         ds['reg_terms'] = (['iteration', 'nr_reg_terms'], self.reg_terms)
         ds['c_terms_description'] = (['iteration'], self.c_terms_description)
         ds['time_needed'] = (['iteration'], self.time_needed)
+        ds['gradient_time_needed'] = (['iteration'], self.gradient_time_needed)
         ds['fct_calls'] = (['iteration'], self.fct_calls)
         ds['unknown_parameters'] = (['iteration', 'nr_unknown_parameters'],
                                     self.unknown_parameters)

agile1d/tests/test_inversion.py

@@ -145,6 +145,7 @@ def test_agile_inversion(self, hef_gdir, control_vars, spinup_options,
         assert data_logger.reg_terms is not None
         assert data_logger.c_terms_description is not None
         assert data_logger.time_needed is not None
+        assert data_logger.gradient_time_needed is not None
         assert data_logger.grads is not None
         assert data_logger.flowlines is not None
         assert data_logger.end_time is not None