run_pretrained_model path relative to config dir

Author: nbcsm

tests/run_pretrained_models.py

@@ -10,6 +10,7 @@
 
 import argparse
 import os
+import re
 import sys
 import tarfile
 import time
@@ -98,8 +99,8 @@ def __init__(self, url, local, make_input, input_names, output_names,
         self.force_input_shape = force_input_shape
         self.skip_tensorflow = skip_tensorflow
 
-    def download_file(self):
-        """Download file from url."""
+    def download_model(self):
+        """Download model from url."""
         cache_dir = Test.cache_dir
         if not os.path.exists(cache_dir):
             os.makedirs(cache_dir)
@@ -193,13 +194,13 @@ def run_test(self, name, backend="caffe2", onnx_file=None, opset=None, extra_ops
 
         # get the model
         if self.url:
-            _, dir_name = self.download_file()
+            _, dir_name = self.download_model()
+            logger.info("Downloaded to %s", dir_name)
             model_path = os.path.join(dir_name, self.local)
         else:
             model_path = self.local
-            dir_name = os.path.dirname(self.local)
-        logger.info("Downloaded to %s", model_path)
 
+        logger.info("Load model from %s", model_path)
         input_names = list(self.input_names.keys())
         outputs = self.output_names
         if self.model_type in ["checkpoint"]:
@@ -324,10 +325,13 @@ def get_args():
     return args
 
 
-def tests_from_yaml(fname):
+def tests_from_yaml(path):
     """Create test class from yaml file."""
+    path = os.path.abspath(path)
+    base_dir = os.path.dirname(path)
+
     tests = {}
-    config = yaml.load(open(fname, 'r').read())
+    config = yaml.load(open(path, 'r').read())
     for k, v in config.items():
         input_func = v.get("input_get")
         input_func = _INPUT_FUNC_MAPPING[input_func]
@@ -337,7 +341,27 @@ def tests_from_yaml(fname):
             if v.get(kw) is not None:
                 kwargs[kw] = v[kw]
 
-        test = Test(v.get("url"), v.get("model"), input_func, v.get("inputs"), v.get("outputs"), **kwargs)
+        # when model is local, non-absolute path is relative to yaml directory
+        url = v.get("url")
+        model = v.get("model")
+        if not url:
+            if not os.path.isabs(model):
+                model = os.path.join(base_dir, model)
+
+        # non-absolute npy file path for np.load is relative to yaml directory
+        input_names = v.get("inputs")
+        for key in list(input_names.keys()):
+            value = input_names[key]
+            if isinstance(value, str):
+                # assume at most 1 match
+                matches = re.findall(r"np\.load\((r?['\"].*?['\"])", value)
+                if matches:
+                    npy_path = matches[0].lstrip('r').strip("'").strip('"')
+                    if not os.path.isabs(npy_path):
+                        abs_npy_path = os.path.join(base_dir, npy_path)
+                        input_names[key] = value.replace(matches[0], "r'{}'".format(abs_npy_path))
+
+        test = Test(url, model, input_func, input_names, v.get("outputs"), **kwargs)
         tests[k] = test
     return tests
 

tests/run_pretrained_models.yaml

@@ -2,15 +2,15 @@
 # simple models for basic functional test
 #
 regression-graphdef:
-  model: tests/models/regression/graphdef/frozen.pb
+  model: models/regression/graphdef/frozen.pb
   input_get: get_ramp
   inputs:
     "X:0": [1]
   outputs:
     - pred:0
 
 regression-checkpoint:
-  model: tests/models/regression/checkpoint/model.meta
+  model: models/regression/checkpoint/model.meta
   model_type: checkpoint
   input_get: get_ramp
   inputs:
@@ -19,7 +19,7 @@ regression-checkpoint:
     - pred:0
 
 regression-saved-model:
-  model: tests/models/regression/saved_model
+  model: models/regression/saved_model
   model_type: saved_model
   input_get: get_ramp
   inputs:
@@ -28,7 +28,7 @@ regression-saved-model:
     - pred:0
 
 saved_model_with_redundant_inputs:
-  model: tests/models/saved_model_with_redundant_inputs
+  model: models/saved_model_with_redundant_inputs
   model_type: saved_model
   input_get: get_ramp
   inputs:
@@ -38,7 +38,7 @@ saved_model_with_redundant_inputs:
     - Add:0
 
 graphdef_with_redundant_inputs:
-  model: tests/models/regression/graphdef/frozen.pb
+  model: models/regression/graphdef/frozen.pb
   input_get: get_ramp
   inputs:
     "X:0": [1, 10]
@@ -47,7 +47,7 @@ graphdef_with_redundant_inputs:
     - Add:0
 
 checkpoint_with_redundant_inputs:
-  model: tests/models/regression/checkpoint/model.meta
+  model: models/regression/checkpoint/model.meta
   model_type: checkpoint
   input_get: get_ramp
   inputs:
@@ -57,15 +57,15 @@ checkpoint_with_redundant_inputs:
     - pred:0
 
 benchtf-fc:
-  model: tests/models/fc-layers/frozen.pb
+  model: models/fc-layers/frozen.pb
   input_get: get_ramp
   inputs:
     "X:0": [1, 784]
   outputs:
     - output:0
 
 benchtf-conv:
-  model: tests/models/conv-layers/frozen.pb
+  model: models/conv-layers/frozen.pb
   input_get: get_ramp
   inputs:
     "X:0": [1, 784]
@@ -74,15 +74,15 @@ benchtf-conv:
 
 benchtf-convbn:
   disabled: true # some if from training isn't removed
-  model: tests/models/convbn-layers/frozen.pb
+  model: models/convbn-layers/frozen.pb
   input_get: get_ramp
   inputs:
     "X:0": [1, 784]
   outputs:
     - output:0
 
 benchtf-ae0:
-  model: tests/models/ae0/frozen.pb
+  model: models/ae0/frozen.pb
   input_get: get_ramp
   inputs:
     "X:0": [1, 784]
@@ -91,7 +91,7 @@ benchtf-ae0:
 
 benchtf-lstm:
   disabled: true
-  model: tests/models/lstm/frozen.pb
+  model: models/lstm/frozen.pb
   input_get: get_ramp
   inputs:
     "X:0": [1, 784]
@@ -100,7 +100,7 @@ benchtf-lstm:
 
 benchtf-gru:
   disabled: true
-  model: tests/models/gru/frozen.pb
+  model: models/gru/frozen.pb
   input_get: get_ramp
   inputs:
     "X:0": [1, 784]