feat: memoize get_task_definition and _get_deps (#477)

This greatly improves performance of get_ancestors for certain inputs.

Author: bhearsum

src/taskgraph/util/taskcluster.py

@@ -3,6 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 
+import copy
 import datetime
 import functools
 import logging
@@ -247,6 +248,7 @@ def get_task_url(task_id, use_proxy=False):
     return task_tmpl.format(task_id)
 
 
+@memoize
 def get_task_definition(task_id, use_proxy=False):
     response = _do_request(get_task_url(task_id, use_proxy))
     return response.json()
@@ -372,13 +374,14 @@ def list_task_group_incomplete_tasks(task_group_id):
             break
 
 
+@memoize
 def _get_deps(task_ids, use_proxy):
     upstream_tasks = {}
     for task_id in task_ids:
         task_def = get_task_definition(task_id, use_proxy)
         upstream_tasks[task_def["metadata"]["name"]] = task_id
 
-        upstream_tasks.update(_get_deps(task_def["dependencies"], use_proxy))
+        upstream_tasks.update(_get_deps(tuple(task_def["dependencies"]), use_proxy))
 
     return upstream_tasks
 
@@ -403,6 +406,6 @@ def get_ancestors(
     for task_id in task_ids:
         task_def = get_task_definition(task_id, use_proxy)
 
-        upstream_tasks.update(_get_deps(task_def["dependencies"], use_proxy))
+        upstream_tasks.update(_get_deps(tuple(task_def["dependencies"]), use_proxy))
 
-    return upstream_tasks
+    return copy.deepcopy(upstream_tasks)

test/test_util_taskcluster.py

@@ -358,6 +358,8 @@ def test_list_task_group_incomplete_tasks(responses, root_url):
 
 
 def test_get_ancestors(responses, root_url):
+    tc.get_task_definition.clear()
+    tc._get_deps.clear()
     base_url = f"{root_url}/api/queue/v1/task"
     responses.add(
         responses.GET,
@@ -431,6 +433,8 @@ def test_get_ancestors(responses, root_url):
 
 
 def test_get_ancestors_string(responses, root_url):
+    tc.get_task_definition.clear()
+    tc._get_deps.clear()
     base_url = f"{root_url}/api/queue/v1/task"
     responses.add(
         responses.GET,