Merge pull request #2704 from Unity-Technologies/hotfix-0.10.1

Ervin T · web-flow · commit 0f5bb4d6ee7a · 2019-10-09T17:34:03.000-07:00
Merge Hotfix 0.10.1
diff --git a/UnitySDK/Assets/ML-Agents/Examples/PushBlock/Brains/PushBlockLearning.asset b/UnitySDK/Assets/ML-Agents/Examples/PushBlock/Brains/PushBlockLearning.asset
@@ -12,13 +12,10 @@ MonoBehaviour:
   m_Name: PushBlockLearning
   m_EditorClassIdentifier: 
   brainParameters:
-    vectorObservationSize: 0
+    vectorObservationSize: 70
     numStackedVectorObservations: 3
     vectorActionSize: 07000000
-    cameraResolutions:
-    - width: 84
-      height: 84
-      blackAndWhite: 0
+    cameraResolutions: []
     vectorActionDescriptions:
     - 
     vectorActionSpaceType: 0
diff --git a/UnitySDK/Assets/ML-Agents/Examples/Pyramids/Prefabs/VisualAreaPyramids.prefab b/UnitySDK/Assets/ML-Agents/Examples/Pyramids/Prefabs/VisualAreaPyramids.prefab
@@ -3196,7 +3196,7 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: b8db44472779248d3be46895c4d562d5, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
-  brain: {fileID: 11400000, guid: 59a04e208fb8a423586adf25bf1fecd0, type: 2}
+  brain: {fileID: 11400000, guid: 60f0ffcd08c3b43a6bdc746cfc0c4059, type: 2}
   agentParameters:
     agentCameras:
     - {fileID: 20712684238256298}
diff --git a/UnitySDK/Assets/ML-Agents/Examples/Pyramids/Scenes/VisualPyramids.unity b/UnitySDK/Assets/ML-Agents/Examples/Pyramids/Scenes/VisualPyramids.unity
@@ -609,9 +609,7 @@ MonoBehaviour:
   broadcastHub:
     broadcastingBrains:
     - {fileID: 11400000, guid: 60f0ffcd08c3b43a6bdc746cfc0c4059, type: 2}
-    m_BrainsToControl:
-    - {fileID: 11400000, guid: 60f0ffcd08c3b43a6bdc746cfc0c4059, type: 2}
-  m_MaxSteps: 0
+    m_BrainsToControl: []
   m_TrainingConfiguration:
     width: 80
     height: 80
diff --git a/config/sac_trainer_config.yaml b/config/sac_trainer_config.yaml
@@ -34,7 +34,6 @@ FoodCollectorLearning:
 
 BouncerLearning:
     normalize: true
-    beta: 0.0
     max_steps: 5.0e5
     num_layers: 2
     hidden_units: 64
@@ -43,7 +42,6 @@ BouncerLearning:
 PushBlockLearning:
     max_steps: 5.0e4
     init_entcoef: 0.05
-    beta: 1.0e-2
     hidden_units: 256
     summary_freq: 2000
     time_horizon: 64
@@ -70,7 +68,6 @@ BigWallJumpLearning:
 StrikerLearning:
     max_steps: 5.0e5
     learning_rate: 1e-3
-    beta: 1.0e-2
     hidden_units: 256
     summary_freq: 2000
     time_horizon: 128
@@ -81,7 +78,6 @@ StrikerLearning:
 GoalieLearning:
     max_steps: 5.0e5
     learning_rate: 1e-3
-    beta: 1.0e-2
     hidden_units: 256
     summary_freq: 2000
     time_horizon: 128
@@ -119,7 +115,6 @@ VisualPyramidsLearning:
     hidden_units: 256
     buffer_init_steps: 1000
     num_layers: 1
-    beta: 1.0e-2
     max_steps: 5.0e5
     buffer_size: 500000
     init_entcoef: 0.01
@@ -216,7 +211,6 @@ HallwayLearning:
     num_layers: 2
     hidden_units: 128
     memory_size: 256
-    beta: 0.0
     init_entcoef: 0.1
     max_steps: 5.0e5
     summary_freq: 1000
@@ -229,7 +223,6 @@ VisualHallwayLearning:
     num_layers: 1
     hidden_units: 128
     memory_size: 256
-    beta: 1.0e-2
     gamma: 0.99
     batch_size: 64
     max_steps: 5.0e5
@@ -243,7 +236,6 @@ VisualPushBlockLearning:
     num_layers: 1
     hidden_units: 128
     memory_size: 256
-    beta: 1.0e-2
     gamma: 0.99
     buffer_size: 1024
     batch_size: 64
diff --git a/gym-unity/setup.py b/gym-unity/setup.py
@@ -11,5 +11,5 @@
     author_email="ML-Agents@unity3d.com",
     url="https://github.com/Unity-Technologies/ml-agents",
     packages=find_packages(),
-    install_requires=["gym", "mlagents_envs==0.10.0"],
+    install_requires=["gym", "mlagents_envs==0.10.1"],
 )
diff --git a/markdown-link-check.config.json b/markdown-link-check.config.json
@@ -7,6 +7,10 @@
         {
             "pattern": "^https://developer.nvidia.com/compute/machine-learning/cudnn/secure",
             "comment": "Requires login"
+        },
+        {
+            "pattern": "^https?://bair.berkeley.edu",
+            "comment": "Temporary berkeley outage"
         }
     ]
 }
diff --git a/ml-agents-envs/setup.py b/ml-agents-envs/setup.py
@@ -5,7 +5,7 @@
 
 setup(
     name="mlagents_envs",
-    version="0.10.0",
+    version="0.10.1",
     description="Unity Machine Learning Agents Interface",
     url="https://github.com/Unity-Technologies/ml-agents",
     author="Unity Technologies",
diff --git a/ml-agents/mlagents/trainers/bc/models.py b/ml-agents/mlagents/trainers/bc/models.py
@@ -40,7 +40,7 @@ def __init__(
             for size in self.act_size:
                 policy_branches.append(
                     tf.layers.dense(
-                        hidden,
+                        hidden_reg,
                         size,
                         activation=None,
                         use_bias=False,
diff --git a/ml-agents/mlagents/trainers/bc/trainer.py b/ml-agents/mlagents/trainers/bc/trainer.py
@@ -129,11 +129,12 @@ def update_policy(self):
             len(self.demonstration_buffer.update_buffer["actions"]) // self.n_sequences,
             self.batches_per_epoch,
         )
-        for i in range(num_batches):
+
+        batch_size = self.n_sequences * self.policy.sequence_length
+
+        for i in range(0, num_batches * batch_size, batch_size):
             update_buffer = self.demonstration_buffer.update_buffer
-            start = i * self.n_sequences
-            end = (i + 1) * self.n_sequences
-            mini_batch = update_buffer.make_mini_batch(start, end)
+            mini_batch = update_buffer.make_mini_batch(i, i + batch_size)
             run_out = self.policy.update(mini_batch, self.n_sequences)
             loss = run_out["policy_loss"]
             batch_losses.append(loss)
diff --git a/ml-agents/mlagents/trainers/ppo/policy.py b/ml-agents/mlagents/trainers/ppo/policy.py
@@ -1,10 +1,10 @@
 import logging
 import numpy as np
-from typing import Any, Dict
+from typing import Any, Dict, Optional
 import tensorflow as tf
 
 from mlagents.envs.timers import timed
-from mlagents.envs.brain import BrainInfo
+from mlagents.envs.brain import BrainInfo, BrainParameters
 from mlagents.trainers.models import EncoderType, LearningRateSchedule
 from mlagents.trainers.ppo.models import PPOModel
 from mlagents.trainers.tf_policy import TFPolicy
@@ -17,7 +17,14 @@
 
 
 class PPOPolicy(TFPolicy):
-    def __init__(self, seed, brain, trainer_params, is_training, load):
+    def __init__(
+        self,
+        seed: int,
+        brain: BrainParameters,
+        trainer_params: Dict[str, Any],
+        is_training: bool,
+        load: bool,
+    ):
         """
         Policy for Proximal Policy Optimization Networks.
         :param seed: Random seed.
@@ -29,8 +36,8 @@ def __init__(self, seed, brain, trainer_params, is_training, load):
         super().__init__(seed, brain, trainer_params)
 
         reward_signal_configs = trainer_params["reward_signals"]
-        self.inference_dict = {}
-        self.update_dict = {}
+        self.inference_dict: Dict[str, tf.Tensor] = {}
+        self.update_dict: Dict[str, tf.Tensor] = {}
         self.stats_name_to_update_name = {
             "Losses/Value Loss": "value_loss",
             "Losses/Policy Loss": "policy_loss",
@@ -42,6 +49,7 @@ def __init__(self, seed, brain, trainer_params, is_training, load):
         self.create_reward_signals(reward_signal_configs)
 
         with self.graph.as_default():
+            self.bc_module: Optional[BCModule] = None
             # Create pretrainer if needed
             if "pretraining" in trainer_params:
                 BCModule.check_config(trainer_params["pretraining"])
@@ -52,8 +60,6 @@ def __init__(self, seed, brain, trainer_params, is_training, load):
                     default_num_epoch=trainer_params["num_epoch"],
                     **trainer_params["pretraining"],
                 )
-            else:
-                self.bc_module = None
 
         if load:
             self._load_graph()
diff --git a/ml-agents/mlagents/trainers/ppo/trainer.py b/ml-agents/mlagents/trainers/ppo/trainer.py
@@ -229,7 +229,16 @@ def update_policy(self):
             mean_return=float(np.mean(self.cumulative_returns_since_policy_update)),
         )
         self.cumulative_returns_since_policy_update = []
-        batch_size = self.trainer_parameters["batch_size"]
+
+        # Make sure batch_size is a multiple of sequence length. During training, we
+        # will need to reshape the data into a batch_size x sequence_length tensor.
+        batch_size = (
+            self.trainer_parameters["batch_size"]
+            - self.trainer_parameters["batch_size"] % self.policy.sequence_length
+        )
+        # Make sure there is at least one sequence
+        batch_size = max(batch_size, self.policy.sequence_length)
+
         n_sequences = max(
             int(self.trainer_parameters["batch_size"] / self.policy.sequence_length), 1
         )
diff --git a/ml-agents/mlagents/trainers/sac/models.py b/ml-agents/mlagents/trainers/sac/models.py
@@ -556,10 +556,13 @@ def create_memory_ins(self, m_size):
         )
         # We assume m_size is divisible by 4
         # Create the non-Policy inputs
+        # Use a default placeholder here so nothing has to be provided during
+        # Barracuda inference. Note that the default value is just the tiled input
+        # for the policy, which is thrown away.
         three_fourths_m_size = m_size * 3 // 4
-        self.other_memory_in = tf.placeholder(
+        self.other_memory_in = tf.placeholder_with_default(
+            input=tf.tile(self.inference_memory_in, [1, 3]),
             shape=[None, three_fourths_m_size],
-            dtype=tf.float32,
             name="other_recurrent_in",
         )
 
diff --git a/ml-agents/mlagents/trainers/sac/policy.py b/ml-agents/mlagents/trainers/sac/policy.py
@@ -1,5 +1,5 @@
 import logging
-from typing import Dict, Any
+from typing import Dict, Any, Optional
 import numpy as np
 import tensorflow as tf
 
@@ -58,6 +58,7 @@ def __init__(
 
         with self.graph.as_default():
             # Create pretrainer if needed
+            self.bc_module: Optional[BCModule] = None
             if "pretraining" in trainer_params:
                 BCModule.check_config(trainer_params["pretraining"])
                 self.bc_module = BCModule(
@@ -74,8 +75,6 @@ def __init__(
                         "Pretraining: Samples Per Update is not a valid setting for SAC."
                     )
                     self.bc_module.samples_per_update = 1
-            else:
-                self.bc_module = None
 
         if load:
             self._load_graph()
diff --git a/ml-agents/mlagents/trainers/sac/trainer.py b/ml-agents/mlagents/trainers/sac/trainer.py
@@ -13,7 +13,6 @@
 from mlagents.envs.brain import AllBrainInfo
 from mlagents.envs.action_info import ActionInfoOutputs
 from mlagents.envs.timers import timed
-from mlagents.trainers.buffer import Buffer
 from mlagents.trainers.sac.policy import SACPolicy
 from mlagents.trainers.rl_trainer import RLTrainer, AllRewardsOutput
 
@@ -121,7 +120,7 @@ def save_replay_buffer(self) -> None:
         with open(filename, "wb") as file_object:
             self.training_buffer.update_buffer.save_to_file(file_object)
 
-    def load_replay_buffer(self) -> Buffer:
+    def load_replay_buffer(self) -> None:
         """
         Loads the last saved replay buffer from a file.
         """
diff --git a/ml-agents/mlagents/trainers/tests/mock_brain.py b/ml-agents/mlagents/trainers/tests/mock_brain.py
@@ -212,6 +212,16 @@ def create_mock_3dball_brain():
     return mock_brain
 
 
+def create_mock_pushblock_brain():
+    mock_brain = create_mock_brainparams(
+        vector_action_space_type="discrete",
+        vector_action_space_size=[7],
+        vector_observation_space_size=70,
+    )
+    mock_brain.brain_name = "PushblockLearning"
+    return mock_brain
+
+
 def create_mock_banana_brain():
     mock_brain = create_mock_brainparams(
         number_visual_observations=1,
diff --git a/ml-agents/mlagents/trainers/tests/test_barracuda_converter.py b/ml-agents/mlagents/trainers/tests/test_barracuda_converter.py
@@ -1,7 +1,10 @@
 import os
+import yaml
+import pytest
 import tempfile
 
 import mlagents.trainers.tensorflow_to_barracuda as tf2bc
+from mlagents.trainers.tests.test_bc import create_bc_trainer
 
 
 def test_barracuda_converter():
@@ -24,3 +27,29 @@ def test_barracuda_converter():
 
     # cleanup
     os.remove(tmpfile)
+
+
+@pytest.fixture
+def bc_dummy_config():
+    return yaml.safe_load(
+        """
+            hidden_units: 32
+            learning_rate: 3.0e-4
+            num_layers: 1
+            use_recurrent: false
+            sequence_length: 32
+            memory_size: 64
+            batches_per_epoch: 1
+            batch_size: 64
+            summary_freq: 2000
+            max_steps: 4000
+            """
+    )
+
+
+@pytest.mark.parametrize("use_lstm", [False, True], ids=["nolstm", "lstm"])
+@pytest.mark.parametrize("use_discrete", [True, False], ids=["disc", "cont"])
+def test_bc_export(bc_dummy_config, use_lstm, use_discrete):
+    bc_dummy_config["use_recurrent"] = use_lstm
+    trainer, env = create_bc_trainer(bc_dummy_config, use_discrete)
+    trainer.export_model()
diff --git a/ml-agents/mlagents/trainers/tests/test_bc.py b/ml-agents/mlagents/trainers/tests/test_bc.py
@@ -32,10 +32,18 @@ def dummy_config():
     )
 
 
-def create_bc_trainer(dummy_config):
+def create_bc_trainer(dummy_config, is_discrete=False):
     mock_env = mock.Mock()
-    mock_brain = mb.create_mock_3dball_brain()
-    mock_braininfo = mb.create_mock_braininfo(num_agents=12, num_vector_observations=8)
+    if is_discrete:
+        mock_brain = mb.create_mock_pushblock_brain()
+        mock_braininfo = mb.create_mock_braininfo(
+            num_agents=12, num_vector_observations=70
+        )
+    else:
+        mock_brain = mb.create_mock_3dball_brain()
+        mock_braininfo = mb.create_mock_braininfo(
+            num_agents=12, num_vector_observations=8
+        )
     mb.setup_mock_unityenvironment(mock_env, mock_brain, mock_braininfo)
     env = mock_env()
 
diff --git a/ml-agents/mlagents/trainers/tests/test_ppo.py b/ml-agents/mlagents/trainers/tests/test_ppo.py
diff --git a/ml-agents/mlagents/trainers/trainer_metrics.py b/ml-agents/mlagents/trainers/trainer_metrics.py
diff --git a/ml-agents/mlagents/trainers/trainer_util.py b/ml-agents/mlagents/trainers/trainer_util.py
diff --git a/ml-agents/setup.py b/ml-agents/setup.py

Original file line number	Diff line number	Diff line change
`@@ -11,5 +11,5 @@`
`11`	`11`	`author_email="[email protected]",`
`12`	`12`	`url="https://github.com/Unity-Technologies/ml-agents",`
`13`	`13`	`packages=find_packages(),`
`14`		`- install_requires=["gym", "mlagents_envs==0.10.0"],`
	`14`	`+ install_requires=["gym", "mlagents_envs==0.10.1"],`
`15`	`15`	`)`
Original file line number	Diff line number	Diff line change
`@@ -7,6 +7,10 @@`
`7`	`7`	`{`
`8`	`8`	`"pattern": "^https://developer.nvidia.com/compute/machine-learning/cudnn/secure",`
`9`	`9`	`"comment": "Requires login"`
	`10`	`+ },`
	`11`	`+ {`
	`12`	`+ "pattern": "^https?://bair.berkeley.edu",`
	`13`	`+ "comment": "Temporary berkeley outage"`
`10`	`14`	`}`
`11`	`15`	`]`
`12`	`16`	`}`