fixed error in critic constructor call

Author: philtabor

ReinforcementLearning/PolicyGradient/DDPG/tensorflow2/pendulum/ddpg_tf2.py

@@ -1,14 +1,14 @@
-import numpy as np
 import tensorflow as tf
 import tensorflow.keras as keras
 from tensorflow.keras.optimizers import Adam
 from buffer import ReplayBuffer
 from networks import ActorNetwork, CriticNetwork
 
+
 class Agent:
     def __init__(self, input_dims, alpha=0.001, beta=0.002, env=None,
-            gamma=0.99, n_actions=2, max_size=1000000, tau=0.005, 
-            fc1=400, fc2=300, batch_size=64, noise=0.1):
+                 gamma=0.99, n_actions=2, max_size=1000000, tau=0.005,
+                 fc1=400, fc2=300, batch_size=64, noise=0.1):
         self.gamma = gamma
         self.tau = tau
         self.memory = ReplayBuffer(max_size, input_dims, n_actions)
@@ -17,11 +17,12 @@ def __init__(self, input_dims, alpha=0.001, beta=0.002, env=None,
         self.noise = noise
         self.max_action = env.action_space.high[0]
         self.min_action = env.action_space.low[0]
-        
+
         self.actor = ActorNetwork(n_actions=n_actions, name='actor')
-        self.critic = CriticNetwork(n_actions=n_actions, name='critic')
-        self.target_actor = ActorNetwork(n_actions=n_actions, name='target_actor')
-        self.target_critic = CriticNetwork(n_actions=n_actions, name='target_critic')
+        self.critic = CriticNetwork(name='critic')
+        self.target_actor = ActorNetwork(n_actions=n_actions,
+                                         name='target_actor')
+        self.target_critic = CriticNetwork(name='target_critic')
 
         self.actor.compile(optimizer=Adam(learning_rate=alpha))
         self.critic.compile(optimizer=Adam(learning_rate=beta))
@@ -68,8 +69,8 @@ def choose_action(self, observation, evaluate=False):
         actions = self.actor(state)
         if not evaluate:
             actions += tf.random.normal(shape=[self.n_actions],
-                    mean=0.0, stddev=self.noise)
-        # note that if the environment has an action > 1, we have to multiply by
+                                        mean=0.0, stddev=self.noise)
+        # note that if the env has an action > 1, we have to multiply by
         # max action at some point
         actions = tf.clip_by_value(actions, self.min_action, self.max_action)
 
@@ -80,7 +81,7 @@ def learn(self):
             return
 
         state, action, reward, new_state, done = \
-                self.memory.sample_buffer(self.batch_size)
+            self.memory.sample_buffer(self.batch_size)
 
         states = tf.convert_to_tensor(state, dtype=tf.float32)
         states_ = tf.convert_to_tensor(new_state, dtype=tf.float32)
@@ -92,11 +93,11 @@ def learn(self):
             critic_value_ = tf.squeeze(self.target_critic(
                                 states_, target_actions), 1)
             critic_value = tf.squeeze(self.critic(states, actions), 1)
-            target = reward + self.gamma*critic_value_*(1-done)
+            target = rewards + self.gamma*critic_value_*(1-done)
             critic_loss = keras.losses.MSE(target, critic_value)
 
         critic_network_gradient = tape.gradient(critic_loss,
-                                            self.critic.trainable_variables)
+                                                self.critic.trainable_variables)
         self.critic.optimizer.apply_gradients(zip(
             critic_network_gradient, self.critic.trainable_variables))
 
@@ -105,8 +106,8 @@ def learn(self):
             actor_loss = -self.critic(states, new_policy_actions)
             actor_loss = tf.math.reduce_mean(actor_loss)
 
-        actor_network_gradient = tape.gradient(actor_loss, 
-                                    self.actor.trainable_variables)
+        actor_network_gradient = tape.gradient(actor_loss,
+                                               self.actor.trainable_variables)
         self.actor.optimizer.apply_gradients(zip(
             actor_network_gradient, self.actor.trainable_variables))