SERVER-33963 Add task_groups & display_tasks for compile, compile_all, compile_unittests & unittests

Author: Jonathan Abrahams

buildscripts/ciconfig/evergreen.py

@@ -11,7 +11,7 @@
 import yaml
 
 
-class EvergreenProjectConfig(object):
+class EvergreenProjectConfig(object):  # pylint: disable=too-many-instance-attributes
     """Represent an Evergreen project configuration file."""
 
     def __init__(self, path):
@@ -21,8 +21,12 @@ def __init__(self, path):
         self.path = path
         self.tasks = [Task(task_dict) for task_dict in self._conf["tasks"]]
         self._tasks_by_name = {task.name: task for task in self.tasks}
+        self.task_groups = [
+            TaskGroup(task_group_dict) for task_group_dict in self._conf.get("task_groups", [])
+        ]
+        self._task_groups_by_name = {task_group.name: task_group for task_group in self.task_groups}
         self.variants = [
-            Variant(variant_dict, self._tasks_by_name)
+            Variant(variant_dict, self._tasks_by_name, self._task_groups_by_name)
             for variant_dict in self._conf["buildvariants"]
         ]
         self._variants_by_name = {variant.name: variant for variant in self.variants}
@@ -39,6 +43,15 @@ def get_task(self, task_name):
         """Return the task with the given name as a Task instance."""
         return self._tasks_by_name.get(task_name)
 
+    @property
+    def task_group_names(self):
+        """Get the list of task_group names."""
+        return self._task_groups_by_name.keys()
+
+    def get_task_group(self, task_group_name):
+        """Return the task_group with the given name as a Task instance."""
+        return self._task_groups_by_name.get(task_group_name)
+
     @property
     def lifecycle_task_names(self):
         """Get the list of names of the tasks that have not been excluded from test lifecycle."""
@@ -99,17 +112,45 @@ def __str__(self):
         return self.name
 
 
+class TaskGroup(object):
+    """Represent a task_group configuration as found in an Evergreen project configuration file."""
+
+    def __init__(self, conf_dict):
+        """Initialize a TaskGroup from a dictionary containing its configuration."""
+        self.raw = conf_dict
+
+    @property
+    def name(self):
+        """Get the task_group name."""
+        return self.raw["name"]
+
+    @property
+    def tasks(self):
+        """Get the list of task names for task_group."""
+        return self.raw.get("tasks", [])
+
+    def __str__(self):
+        return self.name
+
+
 class Variant(object):
     """Build variant configuration as found in an Evergreen project configuration file."""
 
-    def __init__(self, conf_dict, task_map):
+    def __init__(self, conf_dict, task_map, task_group_map):
         """Initialize Variant."""
         self.raw = conf_dict
         run_on = self.run_on
-        self.tasks = [
-            VariantTask(task_map.get(t["name"]), t.get("distros", run_on), self)
-            for t in conf_dict["tasks"]
-        ]
+        self.tasks = []
+        for task in conf_dict["tasks"]:
+            task_name = task.get("name")
+            if task_name in task_group_map:
+                # A task in conf_dict may be a task_group, containing a list of tasks.
+                for task_in_group in task_group_map.get(task_name).tasks:
+                    self.tasks.append(
+                        VariantTask(task_map.get(task_in_group), task.get("distros", run_on), self))
+            else:
+                self.tasks.append(
+                    VariantTask(task_map.get(task["name"]), task.get("distros", run_on), self))
         self.distro_names = set(run_on)
         for task in self.tasks:
             self.distro_names.update(task.run_on)

buildscripts/tests/ciconfig/evergreen.yml

@@ -9,11 +9,13 @@ functions:
         script: |
           echo "this is a 2nd command in the function!"
           ls
+
   "debug":
     command: shell.exec
     params:
       script: |
         echo "i am a debug function."
+
   "run a task that fails" :
     command: shell.exec
     params:
@@ -36,19 +38,32 @@ functions:
       script: |
         echo "I was called with ${foobar}"
 
-pre:
-  command: shell.exec
-  params:
-    script: |
-      rm -rf src || true
-      echo "pre-task run. JUST ONE COMMAND"
+  "do pre work":
+    command: shell.exec
+    params:
+      script: |
+        rm -rf src || true
+        echo "pre-task run. JUST ONE COMMAND"
 
-post:
-  - command: shell.exec
+  "do post work":
+    command: shell.exec
     params:
       script: |
         echo "post-task run."
         true
+
+  "run timeout work":
+    command: shell.exec
+    params:
+      script: |
+        echo "timeout run."
+        true
+
+pre: &pre
+  - func: "do pre work"
+
+post:
+  - func: "do post work"
   - command: attach.results
     params:
       file_location: src/results.json
@@ -57,6 +72,23 @@ test_lifecycle_excluded_tasks:
 - burn_in_tests
 - no_lifecycle*
 
+task_groups:
+- name: tg_1
+  max_hosts: 1
+  tasks:
+  - compile
+  - passing_test
+  setup_task:
+  - func: "fetch source"
+  teardown_task:
+  - func: "debug"
+  setup_group:
+    *pre
+  teardown_group:
+  - func: "do post work"
+  timeout:
+  - func: "run timeout work"
+
 tasks:
 - name: compile
   depends_on: []
@@ -154,3 +186,10 @@ buildvariants:
   - debian-stretch
   tasks:
   - name: resmoke_task
+- name: amazon
+  display_name: Amazon
+  run_on:
+  - amazon
+  tasks:
+  - name: tg_1
+  - name: timeout_test

buildscripts/tests/ciconfig/test_evergreen.py

@@ -35,6 +35,11 @@ def test_list_tasks(self):
         self.assertIn("no_lifecycle_task", self.conf.task_names)
         self.assertIn("resmoke_task", self.conf.task_names)
 
+    def test_list_task_groups(self):
+        self.assertEqual(1, len(self.conf.task_groups))
+        self.assertEqual(1, len(self.conf.task_group_names))
+        self.assertIn("tg_1", self.conf.task_group_names)
+
     def test_list_lifecycle_task_names(self):
         self.assertEqual(5, len(self.conf.lifecycle_task_names))
         self.assertIn("compile", self.conf.task_names)
@@ -44,11 +49,12 @@ def test_list_lifecycle_task_names(self):
         self.assertIn("resmoke_task", self.conf.task_names)
 
     def test_list_variants(self):
-        self.assertEqual(3, len(self.conf.variants))
-        self.assertEqual(3, len(self.conf.variant_names))
+        self.assertEqual(4, len(self.conf.variants))
+        self.assertEqual(4, len(self.conf.variant_names))
         self.assertIn("osx-108", self.conf.variant_names)
         self.assertIn("ubuntu", self.conf.variant_names)
         self.assertIn("debian", self.conf.variant_names)
+        self.assertIn("amazon", self.conf.variant_names)
 
     def test_get_variant(self):
         variant = self.conf.get_variant("osx-108")
@@ -57,11 +63,12 @@ def test_get_variant(self):
         self.assertEqual("osx-108", variant.name)
 
     def test_list_distro_names(self):
-        self.assertEqual(4, len(self.conf.distro_names))
+        self.assertEqual(5, len(self.conf.distro_names))
         self.assertIn("localtestdistro", self.conf.distro_names)
         self.assertIn("ubuntu1404-test", self.conf.distro_names)
         self.assertIn("pdp-11", self.conf.distro_names)
         self.assertIn("debian-stretch", self.conf.distro_names)
+        self.assertIn("amazon", self.conf.distro_names)
 
 
 class TestTask(unittest.TestCase):
@@ -95,6 +102,34 @@ def test_resmoke_args(self):
         self.assertEqual("core", task.resmoke_suite)
 
 
+class TestTaskGroup(unittest.TestCase):
+    """Unit tests for the TaskGroup class."""
+
+    def test_from_list(self):
+        task_group_dict = {
+            "name": "my_group", "max_hosts": 3, "tasks": ["task1", "task2"], "setup_task": [],
+            "teardown_task": [], "setup_group": [], "teardown_group": [], "timeout": []
+        }
+        task_group = _evergreen.TaskGroup(task_group_dict)
+
+        self.assertEqual("my_group", task_group.name)
+        self.assertEqual(2, len(task_group.tasks))
+        self.assertEqual(task_group_dict, task_group.raw)
+
+    def test_resmoke_args(self):
+        task_dict = {
+            "name":
+                "jsCore", "commands": [{
+                    "func": "run tests",
+                    "vars": {"resmoke_args": "--suites=core --shellWriteMode=commands"}
+                }]
+        }
+        task = _evergreen.Task(task_dict)
+
+        self.assertEqual("--suites=core --shellWriteMode=commands", task.resmoke_args)
+        self.assertEqual("core", task.resmoke_suite)
+
+
 class TestVariant(unittest.TestCase):
     """Unit tests for the Variant class."""
 
@@ -105,13 +140,14 @@ def setUpClass(cls):
     def test_from_dict(self):
         task = _evergreen.Task({"name": "compile"})
         tasks_map = {task.name: task}
+        task_groups_map = {}
         variant_dict = {
             "name": "ubuntu",
             "display_name": "Ubuntu",
             "run_on": ["ubuntu1404-test"],
             "tasks": [{"name": "compile"}],
         }
-        variant = _evergreen.Variant(variant_dict, tasks_map)
+        variant = _evergreen.Variant(variant_dict, tasks_map, task_groups_map)
 
         self.assertEqual("ubuntu", variant.name)
         self.assertEqual("Ubuntu", variant.display_name)
@@ -194,3 +230,8 @@ def test_variant_tasks(self):
         resmoke_task = variant_debian.get_task("resmoke_task")
         self.assertEqual("--suites=somesuite --storageEngine=mmapv1",
                          resmoke_task.combined_resmoke_args)
+
+        # Check for tasks included in task_groups
+        variant_amazon = self.conf.get_variant("amazon")
+        self.assertEqual(3, len(variant_amazon.tasks))
+        self.assertIn("compile", variant_amazon.task_names)