Be resilient to the presence of taskQueueId, projectId (#499)

Taskcluster v41 includes these properties in task definitions.  When
comparing tasks, this normalizes the tasks to always contain `projectId`
and `taskQueueId` and ever `providerId` or `workerType`, following the
same defaults for those properties as Taskcluster itself.

Author: djmitche

src/scriptworker/cot/verify.py

@@ -53,6 +53,8 @@
 )
 from scriptworker.utils import (
     add_enumerable_item_to_dict,
+    add_projectid,
+    add_taskqueueid,
     format_json,
     get_hash,
     get_loggable_url,
@@ -1597,6 +1599,15 @@ def compare_jsone_task_definition(parent_link, rebuilt_definitions):
         compare_definition = remove_empty_keys(compare_definition)
         runtime_definition = remove_empty_keys(parent_link.task)
 
+        # add projectId properties if they do not already appear
+        compare_definition = add_projectid(compare_definition)
+        runtime_definition = add_projectid(runtime_definition)
+
+        # add taskQueueId properties if they do not already appear, and
+        # remove provisionerId / workerType
+        compare_definition = add_taskqueueid(compare_definition)
+        runtime_definition = add_taskqueueid(runtime_definition)
+
         diff = list(dictdiffer.diff(compare_definition, runtime_definition))
         if diff:
             diffs.append(pprint.pformat(diff))

src/scriptworker/utils.py

@@ -854,6 +854,42 @@ def remove_empty_keys(values, remove=({}, None, [], "null")):
     return values
 
 
+# add_projectid {{{1
+def add_projectid(task_def):
+    """Add a projectId property to a task, if none already exists, using
+    the Taskcluster default value of 'none'.
+
+    Args:
+        task_def (dict): the task definition
+
+    Returns:
+        task_def (dict): the task definition, with projectId
+    """
+    return {"projectId": "none", **task_def}
+
+
+# add_taskqueueid {{{1
+def add_taskqueueid(task_def):
+    """Add a taskQueueId property to a task, if none already exists, based
+    on the provisionerId and workerType.  Then remove those two properties.
+
+    Args:
+        task_def (dict): the task definition
+
+    Returns:
+        task_def (dict): the task definition, with taskQueueId
+    """
+    if "taskQueueId" not in task_def or "provisionerId" in task_def or "workerType" in task_def:
+        task_def = task_def.copy()
+    if "taskQueueId" not in task_def:
+        task_def["taskQueueId"] = task_def["provisionerId"] + "/" + task_def["workerType"]
+    if "provisionerId" in task_def:
+        del task_def["provisionerId"]
+    if "workerType" in task_def:
+        del task_def["workerType"]
+    return task_def
+
+
 # get_single_item_from_sequence {{{1
 def get_single_item_from_sequence(
     sequence,