switched final fim inverse to tf linalg inv from pinv

Author: davidsebfischer

batchglm/pkg_constants.py

@@ -34,7 +34,7 @@
 TRUST_REGION_ETA1 = 0.25
 TRUST_REGION_ETA2 = 0.25
 TRUST_REGION_T1 = 0.5  # Fast collapse to avoid trailing.
-TRUST_REGION_T2 = 2.  # Allow expansion if not shrinking.
+TRUST_REGION_T2 = 1.5  # Allow expansion if not shrinking.
 TRUST_REGION_UPPER_BOUND = 1e5
 
 # Convergence hyper-parameters:

batchglm/train/tf/base_glm_all/estimator.py

@@ -329,7 +329,7 @@ def finalize(self):
         a_var = self.session.run(self.model.a_var)
         b_var = self.session.run(self.model.b_var)
         fisher_inv = self.session.run(self.model.fisher_inv)
-        hessians = self.session.run(self.model.hessians)
+        hessian = self.session.run(self.model.hessian)
         jacobian = self.session.run(self.model.gradients)
         log_likelihood = self.session.run(self.model.log_likelihood)
         loss = self.session.run(self.model.loss)
@@ -339,7 +339,7 @@ def finalize(self):
         self.model._a_var = a_var
         self.model._b_var = b_var
         self._fisher_inv = fisher_inv
-        self._hessians = hessians
+        self._hessian = hessian
         self._jacobian = jacobian
         self._log_likelihood = log_likelihood
         self._loss = loss

batchglm/train/tf/base_glm_all/estimator_graph.py

@@ -530,8 +530,8 @@ def __init__(
             )
             self.loss = self.full_data_model.loss_final
             self.log_likelihood = self.full_data_model.log_likelihood_final
-            self.hessians = self.full_data_model.hessians_final
-            self.fisher_inv = op_utils.pinv(-self.full_data_model.hessians_final)  # TODO switch for fim?
+            self.hessian = self.full_data_model.hessians_final
+            self.fisher_inv = tf.linalg.inv(-self.full_data_model.hessians_final)  # TODO switch for fim?
             # Summary statistics on feature-wise model gradients:
             self.gradients = tf.reduce_sum(tf.abs(self.full_data_model.neg_jac_final / num_observations), axis=1)
 

batchglm/train/tf/glm_norm/fim.py

@@ -14,8 +14,7 @@ def _weight_fim_aa(
             loc,
             scale
     ):
-        scalar_one = tf.constant(1, shape=(), dtype=self.dtype)
-        W = tf.square(tf.divide(scalar_one, scale))
+        W = tf.square(tf.divide(tf.ones_like(scale), scale))
 
         return W
 
@@ -24,7 +23,6 @@ def _weight_fim_bb(
             loc,
             scale
     ):
-        scalar_two = tf.constant(2, shape=(), dtype=self.dtype)
-        W = scalar_two * tf.ones_like(loc)
+        W = tf.constant(2, shape=loc.shape, dtype=self.dtype)
 
         return W

batchglm/train/tf/ops.py

@@ -33,26 +33,6 @@ def swap_dims(tensor, axis0, axis1, exec_transpose=True, return_perm=False, name
         return perm1
 
 
-def pinv(matrix, threshold=1e-5):
-    """
-    Calculate the Moore-Penrose pseudo-inverse of the last two dimensions of some matrix.
-
-    E.g. if `matrix` has some shape [..., K, L, M, N], this method will inverse each [M, N] matrix.
-
-    :param matrix: The matrix to invert
-    :param threshold: threshold value
-    :return: the pseudo-inverse of `matrix`
-    """
-
-    s, u, v = tf.linalg.svd(matrix)  # , full_matrices=True, compute_uv=True)
-
-    adj_threshold = tf.reduce_max(s, axis=-1, keepdims=True) * threshold
-    s_inv = tf.where(s > tf.broadcast_to(adj_threshold, s.shape), tf.math.reciprocal(s), tf.zeros_like(s))
-    s_inv = tf.linalg.diag(s_inv)
-
-    return v @ (s_inv @ swap_dims(u, axis0=-1, axis1=-2))
-
-
 def stacked_lstsq(L, b, rcond=1e-10, name="stacked_lstsq"):
     r"""
     Solve `Lx = b`, via SVD least squares cutting of small singular values