Rework test_api testing and fix a bug

Bug was found by the new tests

Author: theotherjimmy

tools/test/test_api/test_api_test.py

@@ -15,127 +15,77 @@
 limitations under the License.
 """
 
-import unittest
+import pytest
 from mock import patch
+from tools.targets import set_targets_json_location
 from tools.test_api import find_tests, build_tests
 
 """
 Tests for test_api.py
 """
 
-class TestApiTests(unittest.TestCase):
+def setUp(self):
     """
-    Test cases for Test Api
+    Called before each test case
+
+    :return:
+    """
+    self.base_dir = 'base_dir'
+    self.target = "K64F"
+    self.toolchain_name = "ARM"
+
+@pytest.mark.parametrize("base_dir", ["base_dir"])
+@pytest.mark.parametrize("target", ["K64F"])
+@pytest.mark.parametrize("toolchain_name", ["ARM"])
+@pytest.mark.parametrize("app_config", ["app_config", None])
+def test_find_tests_app_config(base_dir, target, toolchain_name, app_config):
     """
+    Test find_tests for correct use of app_config
 
-    def setUp(self):
-        """
-        Called before each test case
-
-        :return:
-        """
-        self.base_dir = 'base_dir'
-        self.target = "K64F"
-        self.toolchain_name = "ARM"
-
-    def tearDown(self):
-        """
-        Called after each test case
-
-        :return:
-        """
-        pass
-
-    @patch('tools.test_api.scan_resources')
-    @patch('tools.test_api.prepare_toolchain')
-    def test_find_tests_app_config(self, mock_prepare_toolchain, mock_scan_resources):
-        """
-        Test find_tests for correct use of app_config
-
-        :param mock_prepare_toolchain: mock of function prepare_toolchain
-        :param mock_scan_resources: mock of function scan_resources
-        :return:
-        """
-        app_config = "app_config"
+    :param mock_prepare_toolchain: mock of function prepare_toolchain
+    :param mock_scan_resources: mock of function scan_resources
+    :return:
+    """
+    set_targets_json_location()
+    with patch('tools.test_api.scan_resources') as mock_scan_resources,\
+         patch('tools.test_api.prepare_toolchain') as mock_prepare_toolchain:
         mock_scan_resources().inc_dirs.return_value = []
 
-        find_tests(self.base_dir, self.target, self.toolchain_name, app_config=app_config)
+        find_tests(base_dir, target, toolchain_name, app_config=app_config)
 
         args = mock_prepare_toolchain.call_args
-        self.assertTrue('app_config' in args[1],
-                        "prepare_toolchain was not called with app_config")
-        self.assertEqual(args[1]['app_config'], app_config,
-                         "prepare_toolchain was called with an incorrect app_config")
-
-    @patch('tools.test_api.scan_resources')
-    @patch('tools.test_api.prepare_toolchain')
-    def test_find_tests_no_app_config(self, mock_prepare_toolchain, mock_scan_resources):
-        """
-        Test find_tests correctly deals with no app_config
-
-        :param mock_prepare_toolchain: mock of function prepare_toolchain
-        :param mock_scan_resources: mock of function scan_resources
-        :return:
-        """
-        mock_scan_resources().inc_dirs.return_value = []
+        assert 'app_config' in args[1],\
+            "prepare_toolchain was not called with app_config"
+        assert args[1]['app_config'] == app_config,\
+            "prepare_toolchain was called with an incorrect app_config"
 
-        find_tests(self.base_dir, self.target, self.toolchain_name)
 
-        args = mock_prepare_toolchain.call_args
-        self.assertTrue('app_config' in args[1],
-                        "prepare_toolchain was not called with app_config")
-        self.assertEqual(args[1]['app_config'], None,
-                         "prepare_toolchain was called with an incorrect app_config")
-
-    @patch('tools.test_api.scan_resources')
-    @patch('tools.test_api.build_project')
-    def test_build_tests_app_config(self, mock_build_project, mock_scan_resources):
-        """
-        Test build_tests for correct use of app_config
-
-        :param mock_prepare_toolchain: mock of function prepare_toolchain
-        :param mock_scan_resources: mock of function scan_resources
-        :return:
-        """
-        tests = {'test1': 'test1_path','test2': 'test2_path'}
-        src_paths = ['.']
-        build_path = "build_path"
-        app_config = "app_config"
-        mock_build_project.return_value = "build_project"
-
-        build_tests(tests, src_paths, build_path, self.target, self.toolchain_name,
-                    app_config=app_config)
+@pytest.mark.parametrize("build_path", ["build_path"])
+@pytest.mark.parametrize("target", ["K64F"])
+@pytest.mark.parametrize("toolchain_name", ["ARM"])
+@pytest.mark.parametrize("app_config", ["app_config", None])
+def test_find_tests_app_config(build_path, target, toolchain_name, app_config):
+    """
+    Test find_tests for correct use of app_config
 
-        arg_list = mock_build_project.call_args_list
-        for args in arg_list:
-            self.assertTrue('app_config' in args[1],
-                            "build_tests was not called with app_config")
-            self.assertEqual(args[1]['app_config'], app_config,
-                             "build_tests was called with an incorrect app_config")
-
-    @patch('tools.test_api.scan_resources')
-    @patch('tools.test_api.build_project')
-    def test_build_tests_no_app_config(self, mock_build_project, mock_scan_resources):
-        """
-        Test build_tests correctly deals with no app_config
-
-        :param mock_prepare_toolchain: mock of function prepare_toolchain
-        :param mock_scan_resources: mock of function scan_resources
-        :return:
-        """
-        tests = {'test1': 'test1_path', 'test2': 'test2_path'}
-        src_paths = ['.']
-        build_path = "build_path"
+    :param mock_prepare_toolchain: mock of function prepare_toolchain
+    :param mock_scan_resources: mock of function scan_resources
+    :return:
+    """
+    tests = {'test1': 'test1_path','test2': 'test2_path'}
+    src_paths = ['.']
+    set_targets_json_location()
+    with patch('tools.test_api.scan_resources') as mock_scan_resources,\
+         patch('tools.test_api.build_project') as mock_build_project:
         mock_build_project.return_value = "build_project"
+        mock_scan_resources().inc_dirs.return_value = []
 
-        build_tests(tests, src_paths, build_path, self.target, self.toolchain_name)
+        build_tests(tests, src_paths, build_path, target, toolchain_name,
+                    app_config=app_config)
 
         arg_list = mock_build_project.call_args_list
         for args in arg_list:
-            self.assertTrue('app_config' in args[1],
-                            "build_tests was not called with app_config")
-            self.assertEqual(args[1]['app_config'], None,
-                             "build_tests was called with an incorrect app_config")
-
-if __name__ == '__main__':
-    unittest.main()
+            assert 'app_config' in args[1],\
+                "build_tests was not called with app_config"
+            assert args[1]['app_config'] == app_config,\
+                "build_tests was called with an incorrect app_config"

tools/test_api.py

@@ -2198,9 +2198,10 @@ def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
                         results.remove(r)
 
                         # Take report from the kwargs and merge it into existing report
-                        report_entry = worker_result['kwargs']['report'][target_name][toolchain_name]
-                        for test_key in report_entry.keys():
-                            report[target_name][toolchain_name][test_key] = report_entry[test_key]
+                        if report:
+                            report_entry = worker_result['kwargs']['report'][target_name][toolchain_name]
+                            for test_key in report_entry.keys():
+                                report[target_name][toolchain_name][test_key] = report_entry[test_key]
 
                         # Set the overall result to a failure if a build failure occurred
                         if ('reason' in worker_result and
@@ -2224,7 +2225,8 @@ def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
                             }
 
                             test_key = worker_result['kwargs']['project_id'].upper()
-                            print report[target_name][toolchain_name][test_key][0][0]['output'].rstrip()
+                            if report:
+                                print report[target_name][toolchain_name][test_key][0][0]['output'].rstrip()
                             print 'Image: %s\n' % bin_file
 
                     except: