boltzman fix overflow by np float64; remove offset minus

Author: kengz

rl/memory/prioritized_exp_replay.py

@@ -30,7 +30,7 @@ def __init__(self, env_spec, max_mem_len=10000, e=0.01, alpha=0.6,
     def get_priority(self, error):
         # add min_priority to prevent root of negative = complex
         p = (error + self.e) ** self.alpha
-        assert not np.isnan(p)
+        assert np.isfinite(p)
         return p
 
     def add_exp(self, action, reward, next_state, terminal):

rl/policy/actor_critic.py

@@ -39,19 +39,18 @@ class SoftmaxPolicy(Policy):
     def __init__(self, env_spec,
                  **kwargs):  # absorb generic param without breaking
         super(SoftmaxPolicy, self).__init__(env_spec)
-        self.clip_val = 500
+        self.clip_val = 500.
         log_self(self)
 
     def select_action(self, state):
         agent = self.agent
         state = np.expand_dims(state, axis=0)
         A_score = agent.actor.predict(state)[0]  # extract from batch predict
         assert A_score.ndim == 1
-        A_score = A_score.astype('float32')  # fix precision nan issue
-        A_score = A_score - np.amax(A_score)  # prevent overflow
+        A_score = A_score.astype('float64')  # fix precision overflow
         exp_values = np.exp(
             np.clip(A_score, -self.clip_val, self.clip_val))
-        assert not np.isnan(exp_values).any()
+        assert np.isfinite(exp_values).all()
         probs = np.array(exp_values / np.sum(exp_values))
         probs /= probs.sum()  # renormalize to prevent floating pt error
         action = np.random.choice(agent.env_spec['actions'], p=probs)

rl/policy/boltzmann.py

@@ -18,19 +18,18 @@ def __init__(self, env_spec,
         self.final_tau = final_tau
         self.tau = self.init_tau
         self.exploration_anneal_episodes = exploration_anneal_episodes
-        self.clip_val = 200
+        self.clip_val = 500.
         log_self(self)
 
     def select_action(self, state):
         agent = self.agent
         state = np.expand_dims(state, axis=0)
         Q_state = agent.model.predict(state)[0]  # extract from batch predict
         assert Q_state.ndim == 1
-        Q_state = Q_state.astype('float32')  # fix precision nan issue
-        Q_state = Q_state - np.amax(Q_state)  # prevent overflow
+        Q_state = Q_state.astype('float64')  # fix precision overflow
         exp_values = np.exp(
-            np.clip(Q_state / float(self.tau), -self.clip_val, self.clip_val))
-        assert not np.isnan(exp_values).any()
+            np.clip(Q_state / self.tau, -self.clip_val, self.clip_val))
+        assert np.isfinite(exp_values).all()
         probs = np.array(exp_values / np.sum(exp_values))
         probs /= probs.sum()  # renormalize to prevent floating pt error
         action = np.random.choice(agent.env_spec['actions'], p=probs)
@@ -66,11 +65,10 @@ def select_action(self, state):
         Q_state2 = agent.model_2.predict(state)[0]
         Q_state = Q_state1 + Q_state2
         assert Q_state.ndim == 1
-        Q_state = Q_state.astype('float32')  # fix precision nan issue
-        Q_state = Q_state - np.amax(Q_state)  # prevent overflow
+        Q_state = Q_state.astype('float64')  # fix precision overflow
         exp_values = np.exp(
-            np.clip(Q_state / float(self.tau), -self.clip_val, self.clip_val))
-        assert not np.isnan(exp_values).any()
+            np.clip(Q_state / self.tau, -self.clip_val, self.clip_val))
+        assert np.isfinite(exp_values).all()
         probs = np.array(exp_values / np.sum(exp_values))
         probs /= probs.sum()  # renormalize to prevent floating pt error
         action = np.random.choice(agent.env_spec['actions'], p=probs)