Merge pull request #47 from onnx/gs/onnx-1.2

update test results

Author: guschmue

tests/run_pretrained_models.py

@@ -79,7 +79,7 @@ class Test(object):
     cache_dir = None
 
     def __init__(self, url, local, make_input, input_names, output_names,
-                 disabled=False, more_inputs=None, rtol=0.01, atol=0.):
+                 disabled=False, more_inputs=None, rtol=0.01, atol=0., check_only_shape=False):
         self.url = url
         self.make_input = make_input
         self.local = local
@@ -89,6 +89,7 @@ def __init__(self, url, local, make_input, input_names, output_names,
         self.more_inputs = more_inputs
         self.rtol = rtol
         self.atol = atol
+        self.check_only_shape = check_only_shape
 
     def download_file(self):
         """Download file from url."""
@@ -239,7 +240,12 @@ def run_test(self, name, backend="caffe2", debug=False, onnx_file=None, opset=No
             print("\trun_onnx OK")
 
             try:
-                np.testing.assert_allclose(tf_results, onnx_results, rtol=self.rtol, atol=self.atol)
+                if self.check_only_shape:
+                    for i in range(len(tf_results)):
+                        np.testing.assert_array_equal(tf_results[i].shape, onnx_results[i].shape)
+                else:
+                    for i in range(len(tf_results)):
+                        np.testing.assert_allclose(tf_results[i], onnx_results[i], rtol=self.rtol, atol=self.atol)
                 print("\tResults: OK")
                 return True
             except Exception as ex:
@@ -276,7 +282,7 @@ def tests_from_yaml(fname):
         input_func = v.get("input_get")
         input_func = _INPUT_FUNC_MAPPING[input_func]
         kwargs = {}
-        for kw in ["rtol", "atol", "disabled", "more_inputs"]:
+        for kw in ["rtol", "atol", "disabled", "more_inputs", "check_only_shape"]:
             if v.get(kw) is not None:
                 kwargs[kw] = v[kw]
 

tests/unity.yaml

@@ -1,5 +1,6 @@
 3DBall:
-  # needs RandomNormal
+  # caffe2: needs RandomNormal
+  # onnxmsrtnext: int->float cast fails
   disabled: true
   url: https://github.com/Unity-Technologies/ml-agents/raw/master/unity-environment/Assets/ML-Agents/Examples/3DBall/TFModels/3DBall.bytes
   model: 3DBall.bytes
@@ -32,18 +33,16 @@ BananaIL:
     - action:0
 
 BananaRL:
-  # needs RandomNormal
-  disabled: true
   url: https://github.com/Unity-Technologies/ml-agents/raw/master/unity-environment/Assets/ML-Agents/Examples/BananaCollectors/TFModels/BananaRL.bytes
   model: BananaRL.bytes
+  check_only_shape: true
   input_get: get_random
   inputs:
     "vector_observation:0": [1, 159]
   outputs:
     - action_probs:0
     - value_estimate:0
 
-
 Basic:
   # needs: onehot
   disabled: true
@@ -58,7 +57,8 @@ Basic:
     - value_estimate:0
 
 Bouncer:
-  # needs RandomNormal
+  # caffe2: needs RandomNormal
+  # onnxmsrtnext: int->float cast fails
   disabled: true
   url: https://github.com/Unity-Technologies/ml-agents/raw/master/unity-environment/Assets/ML-Agents/Examples/Bouncer/TFModels/Bouncer.bytes
   model: Bouncer.bytes
@@ -70,7 +70,8 @@ Bouncer:
     - value_estimate:0
 
 crawler:
-  # needs RandomNormal
+  # caffe2: needs RandomNormal
+  # onnxmsrtnext: int->float cast fails
   disabled: true
   url: https://github.com/Unity-Technologies/ml-agents/raw/master/unity-environment/Assets/ML-Agents/Examples/Crawler/TFModels/crawler.bytes
   model: crawler.bytes
@@ -82,10 +83,9 @@ crawler:
     - value_estimate:0
 
 GridWorld_3x3:
-  # needs Multinomial
-  disabled: true
   url: https://github.com/Unity-Technologies/ml-agents/raw/master/unity-environment/Assets/ML-Agents/Examples/GridWorld/TFModels/GridWorld_3x3.bytes
   model: GridWorld_3x3.bytes
+  check_only_shape: true
   input_get: get_random
   inputs:
     "visual_observation_0:0": [1, 84, 84, 3]
@@ -95,11 +95,10 @@ GridWorld_3x3:
     - action:0
 
 GridWorld_5x5:
-  # needs Multinomial
-  disabled: true
   url: https://github.com/Unity-Technologies/ml-agents/raw/master/unity-environment/Assets/ML-Agents/Examples/GridWorld/TFModels/GridWorld_5x5.bytes
   model: GridWorld_5x5.bytes
   input_get: get_random
+  check_only_shape: true
   inputs:
     "visual_observation_0:0": [1, 84, 84, 3]
   outputs:
@@ -115,14 +114,15 @@ Hallway:
   input_get: get_random
   inputs:
     "vector_observation:0": [1, 36]
+    "recurrent_in:0": [1, 256]
+    "sequence_length:0": [256]
+    "prev_action:0": [1]
   outputs:
     - action_probs:0
     - value_estimate:0
     - action:0
 
 PushBlock:
-  # needs: Multinomial
-  disabled: true
   url: https://github.com/Unity-Technologies/ml-agents/raw/master/unity-environment/Assets/ML-Agents/Examples/PushBlock/TFModels/PushBlock.bytes
   model: PushBlock.bytes
   input_get: get_random
@@ -134,7 +134,8 @@ PushBlock:
     - action:0
 
 Reacher:
-  # needs RandomNormal
+  # caffe2: needs RandomNormal
+  # onnxmsrtnext: int->float cast fails
   disabled: true
   url: https://github.com/Unity-Technologies/ml-agents/raw/master/unity-environment/Assets/ML-Agents/Examples/Reacher/TFModels/Reacher.bytes
   model: Reacher.bytes
@@ -146,10 +147,9 @@ Reacher:
     - action:0
 
 Soccer:
-  # needs Multinomial
-  disabled: true
   url: https://github.com/Unity-Technologies/ml-agents/raw/master/unity-environment/Assets/ML-Agents/Examples/Soccer/TFModels/Soccer.bytes
   model: Soccer.bytes
+  check_only_shape: true
   input_get: get_random
   inputs:
     "GoalieBrain/vector_observation:0": [1, 336]
@@ -163,8 +163,6 @@ Soccer:
     - StrikerBrain/value_estimate:0
 
 Tennis:
-  # needs RandomNormal
-  disabled: true
   url: https://github.com/Unity-Technologies/ml-agents/raw/master/unity-environment/Assets/ML-Agents/Examples/Tennis/TFModels/Tennis.bytes
   model: Tennis.bytes
   input_get: get_random
@@ -174,10 +172,9 @@ Tennis:
     - value_estimate:0
 
 WallJump:
-  # needs Multinomial
-  disabled: true
   url: https://github.com/Unity-Technologies/ml-agents/raw/master/unity-environment/Assets/ML-Agents/Examples/WallJump/TFModels/WallJump.bytes
   model: WallJump.bytes
+  check_only_shape: true
   input_get: get_random
   inputs:
     "SmallWallBrain/vector_observation:0": [1, 444]