PPO additions and warnings

* Add linear decay to learning rate for PPO
* Add warning/exception for unsupported brain configurations w/ PPO

Author: awjuliani

python/PPO.ipynb

@@ -105,7 +105,7 @@
     "# Create the Tensorflow model graph\n",
     "ppo_model = create_agent_model(env, lr=learning_rate,\n",
     "                               h_size=hidden_units, epsilon=epsilon,\n",
-    "                               beta=beta)\n",
+    "                               beta=beta, max_step=max_steps)\n",
     "\n",
     "is_continuous = (env.brains[brain_name].action_space_type == \"continuous\")\n",
     "use_observations = (env.brains[brain_name].number_observations > 0)\n",

python/ppo.py

@@ -15,7 +15,7 @@
 
 Options:
   --help                     Show this message.
-  --max-step=<n>             Maximum number of steps to run environment [default: 5e6].
+  --max-steps=<n>             Maximum number of steps to run environment [default: 5e6].
   --run-path=<path>          The sub-directory name for model and summary statistics [default: ppo].
   --load                     Whether to load the model or randomly initialize [default: False].
   --train                    Whether to train model, or only run inference [default: True].
@@ -38,7 +38,7 @@
 print(options)
 
 # General parameters
-max_steps = float(options['--max-step'])
+max_steps = float(options['--max-steps'])
 model_path = './models/{}'.format(str(options['--run-path']))
 summary_path = './summaries/{}'.format(str(options['--run-path']))
 load_model = options['--load']
@@ -69,7 +69,7 @@
 # Create the Tensorflow model graph
 ppo_model = create_agent_model(env, lr=learning_rate,
                                h_size=hidden_units, epsilon=epsilon,
-                               beta=beta)
+                               beta=beta, max_step=max_steps)
 
 is_continuous = (env.brains[brain_name].action_space_type == "continuous")
 use_observations = (env.brains[brain_name].number_observations > 0)

python/ppo/models.py

@@ -2,9 +2,10 @@
 import tensorflow as tf
 import tensorflow.contrib.layers as c_layers
 from tensorflow.python.tools import freeze_graph
+from unityagents import UnityEnvironmentException
 
 
-def create_agent_model(env, lr=1e-4, h_size=128, epsilon=0.2, beta=1e-3):
+def create_agent_model(env, lr=1e-4, h_size=128, epsilon=0.2, beta=1e-3, max_step=5e6):
     """
     Takes a Unity environment and model-specific hyperparameters and returns the
     appropriate PPO agent model for the environment.
@@ -17,16 +18,23 @@ def create_agent_model(env, lr=1e-4, h_size=128, epsilon=0.2, beta=1e-3):
     """
     brain_name = env.brain_names[0]
     if env.brains[brain_name].action_space_type == "continuous":
-        return ContinuousControlModel(lr, env.brains[brain_name].state_space_size,
-                                      env.brains[brain_name].action_space_size, h_size, epsilon, beta)
+        if env.brains[brain_name].number_observations == 0:
+            return ContinuousControlModel(lr, env.brains[brain_name].state_space_size,
+                                          env.brains[brain_name].action_space_size, h_size, epsilon, beta, max_step)
+        else:
+            raise UnityEnvironmentException("There is currently no PPO model which supports both a continuous "
+                                            "action space and camera observations.")
     if env.brains[brain_name].action_space_type == "discrete":
         if env.brains[brain_name].number_observations == 0:
             return DiscreteControlModel(lr, env.brains[brain_name].state_space_size,
-                                        env.brains[brain_name].action_space_size, h_size, epsilon, beta)
+                                        env.brains[brain_name].action_space_size, h_size, epsilon, beta, max_step)
         else:
             brain = env.brains[brain_name]
+            if env.brains[brain_name].state_space_size > 0:
+                print("This brain contains agents with both observations and states. There is currently no PPO model"
+                      "which supports this. Defaulting to Vision-based PPO model.")
             h, w = brain.camera_resolutions[0]['height'], brain.camera_resolutions[0]['height']
-            return VisualDiscreteControlModel(lr, h, w, env.brains[brain_name].action_space_size, h_size, epsilon, beta)
+            return VisualDiscreteControlModel(lr, h, w, env.brains[brain_name].action_space_size, h_size, epsilon, beta, max_step)
 
 
 def save_model(sess, saver, model_path="./", steps=0):
@@ -37,7 +45,7 @@ def save_model(sess, saver, model_path="./", steps=0):
     :param steps: Current number of steps in training process.
     :param saver: Tensorflow saver for session.
     """
-    last_checkpoint = model_path+'/model-'+str(steps)+'.cptk'
+    last_checkpoint = model_path + '/model-' + str(steps) + '.cptk'
     saver.save(sess, last_checkpoint)
     tf.train.write_graph(sess.graph_def, model_path, 'raw_graph_def.pb', as_text=False)
     print("Saved Model")
@@ -61,7 +69,7 @@ def export_graph(model_path, env_name="env", target_nodes="action"):
 
 
 class PPOModel(object):
-    def __init__(self, probs, old_probs, value, entropy, beta, epsilon, lr):
+    def __init__(self, probs, old_probs, value, entropy, beta, epsilon, lr, max_step):
         """
         Creates training-specific Tensorflow ops for PPO models.
         :param probs: Current policy probabilities
@@ -85,15 +93,18 @@ def __init__(self, probs, old_probs, value, entropy, beta, epsilon, lr):
 
         self.loss = self.policy_loss + self.value_loss - beta * tf.reduce_mean(entropy)
 
-        optimizer = tf.train.AdamOptimizer(learning_rate=lr)
+        self.global_step = tf.Variable(0, trainable=False, name='global_step', dtype=tf.int32)
+        self.learning_rate = tf.train.polynomial_decay(lr, self.global_step,
+                                                       max_step, 1e-10,
+                                                       power=1.0)
+        optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate)
         self.update_batch = optimizer.minimize(self.loss)
 
-        self.global_step = tf.Variable(0, trainable=False, name='global_step', dtype=tf.int32)
-        self.increment_step = tf.assign(self.global_step, self.global_step+1)
+        self.increment_step = tf.assign(self.global_step, self.global_step + 1)
 
 
 class ContinuousControlModel(PPOModel):
-    def __init__(self, lr, s_size, a_size, h_size, epsilon, beta):
+    def __init__(self, lr, s_size, a_size, h_size, epsilon, beta, max_step):
         """
         Creates Continuous Control Actor-Critic model.
         :param s_size: State-space size
@@ -127,11 +138,11 @@ def __init__(self, lr, s_size, a_size, h_size, epsilon, beta):
 
         self.old_probs = tf.placeholder(shape=[None, a_size], dtype=tf.float32, name='old_probabilities')
 
-        PPOModel.__init__(self, self.probs, self.old_probs, self.value, self.entropy, 0.0, epsilon, lr)
+        PPOModel.__init__(self, self.probs, self.old_probs, self.value, self.entropy, 0.0, epsilon, lr, max_step)
 
 
 class DiscreteControlModel(PPOModel):
-    def __init__(self, lr, s_size, a_size, h_size, epsilon, beta):
+    def __init__(self, lr, s_size, a_size, h_size, epsilon, beta, max_step):
         """
         Creates Discrete Control Actor-Critic model.
         :param s_size: State-space size
@@ -158,11 +169,11 @@ def __init__(self, lr, s_size, a_size, h_size, epsilon, beta):
         self.old_responsible_probs = tf.reduce_sum(self.old_probs * self.selected_actions, axis=1)
 
         PPOModel.__init__(self, self.responsible_probs, self.old_responsible_probs,
-                          self.value, self.entropy, beta, epsilon, lr)
+                          self.value, self.entropy, beta, epsilon, lr, max_step)
 
 
 class VisualDiscreteControlModel(PPOModel):
-    def __init__(self, lr, o_size_h, o_size_w, a_size, h_size, epsilon, beta):
+    def __init__(self, lr, o_size_h, o_size_w, a_size, h_size, epsilon, beta, max_step):
         """
         Creates Discrete Control Actor-Critic model for use with visual observations (images).
         :param o_size_h: Observation height.
@@ -194,4 +205,4 @@ def __init__(self, lr, o_size_h, o_size_w, a_size, h_size, epsilon, beta):
         self.old_responsible_probs = tf.reduce_sum(self.old_probs * self.selected_actions, axis=1)
 
         PPOModel.__init__(self, self.responsible_probs, self.old_responsible_probs,
-                          self.value, self.entropy, beta, epsilon, lr)
+                          self.value, self.entropy, beta, epsilon, lr, max_step)

python/ppo/trainer.py

@@ -17,7 +17,7 @@ def __init__(self, ppo_model, sess, info, is_continuous, use_observations):
         self.model = ppo_model
         self.sess = sess
         stats = {'cumulative_reward': [], 'episode_length': [], 'value_estimate': [],
-                 'entropy': [], 'value_loss': [], 'policy_loss': []}
+                 'entropy': [], 'value_loss': [], 'policy_loss': [], 'learning_rate': []}
         self.stats = stats
 
         self.training_buffer = vectorize_history(empty_local_history({}))
@@ -45,11 +45,13 @@ def take_action(self, info, env, brain_name):
                          self.model.batch_size: len(info.states)}
         else:
             feed_dict = {self.model.state_in: info.states, self.model.batch_size: len(info.states)}
-        actions, a_dist, value, ent = self.sess.run([self.model.output, self.model.probs,
-                                                     self.model.value, self.model.entropy],
-                                                    feed_dict=feed_dict)
+        actions, a_dist, value, ent, learn_rate = self.sess.run([self.model.output, self.model.probs,
+                                                                 self.model.value, self.model.entropy,
+                                                                 self.model.learning_rate],
+                                                                feed_dict=feed_dict)
         self.stats['value_estimate'].append(value)
         self.stats['entropy'].append(ent)
+        self.stats['learning_rate'].append(learn_rate)
         new_info = env.step(actions, value={brain_name: value})[brain_name]
         self.add_experiences(info, new_info, epsi, actions, a_dist, value)
         return new_info

python/setup.py

@@ -7,7 +7,7 @@
     required = f.read().splitlines()
 
 setup(name='unityagents',
-      version='0.1',
+      version='0.1.1',
       description='Unity Machine Learning Agents',
       license='Apache License 2.0',
       author='Unity Technologies',