NaN values getting stored as RMSE error in lcurve.out file

[AnuragKr]

Hello,

I have tried to change a code of fitting network, replacing the feed-forward neural network composition with RBF (radial basis function) neural network with a single hidden layer. After making changes I am getting training and validation error values as Nan.

I want to know why it is happening, at least some error should come if it is compiling correctly. Please guide me on what other changes I need to make sure to make it work.

Code snippet for --- ener.py

@cast_precision
    def _build_lower(
            self,
            start_index,
            natoms,
            inputs,
            fparam = None,
            aparam = None,
            bias_atom_e = 0.0,
            suffix = '',
            reuse = None
    ):
        # cut-out inputs
        inputs_i = tf.slice (inputs,
                             [ 0, start_index, 0],
                             [-1, natoms, -1] )
        inputs_i = tf.reshape(inputs_i, [-1, self.dim_descrpt])
        layer = inputs_i
        print(f'Layer Shape : {layer.shape}')
        if fparam is not None:
            ext_fparam = tf.tile(fparam, [1, natoms])
            ext_fparam = tf.reshape(ext_fparam, [-1, self.numb_fparam])
            ext_fparam = tf.cast(ext_fparam,self.fitting_precision)
            layer = tf.concat([layer, ext_fparam], axis = 1)
        if aparam is not None:
            ext_aparam = tf.slice(aparam,
                                  [ 0, start_index      * self.numb_aparam],
                                  [-1, natoms * self.numb_aparam])
            ext_aparam = tf.reshape(ext_aparam, [-1, self.numb_aparam])
            ext_aparam = tf.cast(ext_aparam,self.fitting_precision)
            layer = tf.concat([layer, ext_aparam], axis = 1)

        if nvnmd_cfg.enable:
            one_layer = one_layer_nvnmd
        else:
            one_layer = one_layer_deepmd

        final_layer = rbf_layer_deepmd(
                    layer,
                    self.n_neuron[0],
                    name='rbf_layer_'+suffix,
                    reuse=reuse,
                    seed = self.seed,
                    activation_fn = self.fitting_activation_fn,
                    precision = self.fitting_precision,
                    trainable = self.trainable[0],
                    uniform_seed = self.uniform_seed,
                    initial_variables = self.fitting_net_variables,
                    mixed_prec = self.mixed_prec)
        if (not self.uniform_seed) and (self.seed is not None): self.seed += self.seed_shift

        return final_layer

Code snippet for --- network.py

def rbf_layer(inputs,
              outputs_size,
              activation_fn=tf.nn.tanh,
              precision = GLOBAL_TF_FLOAT_PRECISION,
              stddev=1.0,
              bavg=0.0,
              name='linear',
              scope='',
              reuse=None,
              seed=None,
              use_timestep = False,
              trainable = True,
              useBN = False,
              uniform_seed = False,
              initial_variables = None,
              mixed_prec = None,
              final_layer = False):
 if mixed_prec is not None and final_layer:
        inputs = tf.cast(inputs, get_precision(mixed_prec['output_prec']))
    with tf.variable_scope(name, reuse=reuse):
        shape = inputs.get_shape().as_list()
        w_initializer  = tf.random_normal_initializer(
                            stddev=stddev / np.sqrt(shape[1] + outputs_size),
                            seed=seed if (seed is None or uniform_seed) else seed + 0)
        c_initializer  = tf.random_uniform_initializer(
                            minval=-0.0, maxval=1.0,
                            seed=seed if (seed is None or uniform_seed) else seed + 2)
        b_initializer  = tf.random_normal_initializer(
                            stddev=stddev,
                            mean=bavg,
                            seed=seed if (seed is None or uniform_seed) else seed + 1)
        delta_initializer  = tf.random_normal_initializer(
                            stddev=stddev,
                            seed=seed if (seed is None or uniform_seed) else seed + 1)

        if initial_variables is not None:
            w_initializer = tf.constant_initializer(initial_variables[scope + name + '/matrix'])
            c_initializer = tf.constant_initializer(initial_variables[scope + name + '/center'])
            b_initializer = tf.constant_initializer(initial_variables[scope + name + '/bias'])
            delta_initializer = tf.constant_initializer(initial_variables[scope + name + '/delta'])

        w = tf.get_variable('matrix',
                            [outputs_size, 1],
                            precision,
                            w_initializer,
                            trainable = trainable)
        variable_summaries(w, 'matrix')


c = tf.get_variable('center',
                            [outputs_size,shape[1]],
                            precision,
                            c_initializer,
                            trainable = trainable)

        b = tf.get_variable('bias',
                            [1],
                            precision,
                            b_initializer,
                            trainable = trainable)
        variable_summaries(b, 'bias')

        delta = tf.get_variable('delta',
                            [outputs_size],
                            precision,
                            delta_initializer,
                            trainable = trainable)

        if mixed_prec is not None and not final_layer:
            inputs = tf.cast(inputs, get_precision(mixed_prec['compute_prec']))
            w = tf.cast(w, get_precision(mixed_prec['compute_prec']))
            b = tf.cast(b, get_precision(mixed_prec['compute_prec']))
            c = tf.cast(c, get_precision(mixed_prec['compute_prec']))
            delta = tf.cast(delta, get_precision(mixed_prec['compute_prec']))

        C = tf.expand_dims(c, -1)  # inserts a dimension of 1
        H = tf.transpose(C - tf.transpose(inputs))  # matrix of differences
        RBF_OUT = tf.exp(-delta * tf.math.reduce_sum(tf.math.square(H), axis=1))

        hidden = tf.matmul(RBF_OUT,w) + b

        if activation_fn != None:
            hidden = tf.reshape(activation_fn(hidden), [-1, outputs_size])

        if mixed_prec is not None:
            hidden = tf.cast(hidden, get_precision(mixed_prec['output_prec']))
        return hidden

[AnuragKr]

I changed the c_initalizer with normal distribution , it's now working.