BREAKING CHANGE: Delete taskgraph.util.memoize (fixes #491)

Author: bhearsum

docs/reference/source/taskgraph.util.rst

@@ -60,14 +60,6 @@ taskgraph.util.keyed\_by module
    :undoc-members:
    :show-inheritance:
 
-taskgraph.util.memoize module
------------------------------
-
-.. automodule:: taskgraph.util.memoize
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
 taskgraph.util.parameterization module
 --------------------------------------
 

src/taskgraph/actions/registry.py

@@ -3,6 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 
+import functools
 import json
 from collections import namedtuple
 from types import FunctionType
@@ -13,7 +14,6 @@
 from taskgraph.config import load_graph_config
 from taskgraph.parameters import Parameters
 from taskgraph.util import hash, taskcluster, yaml
-from taskgraph.util.memoize import memoize
 from taskgraph.util.python_path import import_sibling_modules
 
 actions = []
@@ -31,13 +31,13 @@ def is_json(data):
     return True
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def read_taskcluster_yml(filename):
-    """Load and parse .taskcluster.yml, memoized to save some time"""
+    """Load and parse .taskcluster.yml, cached to save some time"""
     return yaml.load_yaml(filename)
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def hash_taskcluster_yml(filename):
     """
     Generate a hash of the given .taskcluster.yml.  This is the first 10 digits

src/taskgraph/transforms/base.py

@@ -3,6 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 
+import functools
 import re
 from dataclasses import dataclass, field
 from typing import Dict, List, Union
@@ -11,7 +12,6 @@
 
 from ..config import GraphConfig
 from ..parameters import Parameters
-from ..util.memoize import memoize
 from ..util.schema import Schema, validate_schema
 
 
@@ -58,7 +58,7 @@ class TransformConfig:
     write_artifacts: bool
 
     @property
-    @memoize
+    @functools.lru_cache(maxsize=None)
     def repo_configs(self):
         repositories = self.graph_config["taskgraph"]["repositories"]
         if len(repositories) == 1:

src/taskgraph/transforms/task.py

@@ -9,6 +9,7 @@
 """
 
 
+import functools
 import hashlib
 import os
 import re
@@ -23,7 +24,6 @@
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.hash import hash_path
 from taskgraph.util.keyed_by import evaluate_keyed_by
-from taskgraph.util.memoize import memoize
 from taskgraph.util.schema import (
     OptimizationSchema,
     Schema,
@@ -43,7 +43,7 @@
 )
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def _run_task_suffix():
     """String to append to cache names under control of run-task."""
     return hash_path(RUN_TASK)[0:20]
@@ -214,14 +214,14 @@ def get_branch_rev(config):
     return config.params["head_rev"]
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def get_default_priority(graph_config, project):
     return evaluate_keyed_by(
         graph_config["task-priority"], "Graph Config", {"project": project}
     )
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def get_default_deadline(graph_config, project):
     return evaluate_keyed_by(
         graph_config["task-deadline-after"], "Graph Config", {"project": project}

src/taskgraph/util/docker.py

@@ -3,14 +3,14 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 
+import functools
 import hashlib
 import io
 import os
 import re
 from typing import Optional
 
 from taskgraph.util.archive import create_tar_gz_from_files
-from taskgraph.util.memoize import memoize
 
 IMAGE_DIR = os.path.join(".", "taskcluster", "docker")
 
@@ -205,7 +205,7 @@ def stream_context_tar(topsrcdir, context_dir, out_file, image_name=None, args=N
     return writer.hexdigest()
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def image_paths():
     """Return a map of image name to paths containing their Dockerfile."""
     config = load_yaml("taskcluster", "kinds", "docker-image", "kind.yml")
@@ -222,7 +222,7 @@ def image_path(name):
     return os.path.join(IMAGE_DIR, name)
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def parse_volumes(image):
     """Parse VOLUME entries from a Dockerfile for an image."""
     volumes = set()

src/taskgraph/util/hash.py

@@ -2,14 +2,14 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import functools
 import hashlib
 from pathlib import Path
 
 from taskgraph.util import path as mozpath
-from taskgraph.util.memoize import memoize
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def hash_path(path):
     """Hash a single file.
 
@@ -44,13 +44,13 @@ def hash_paths(base_path, patterns):
     return h.hexdigest()
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def _find_matching_files(base_path, pattern):
     files = _get_all_files(base_path)
     return [path for path in files if mozpath.match(path, pattern)]
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def _get_all_files(base_path):
     return [
         mozpath.normsep(str(path))

src/taskgraph/util/memoize.py

@@ -16,7 +16,6 @@
 
 from taskgraph.task import Task
 from taskgraph.util import yaml
-from taskgraph.util.memoize import memoize
 
 logger = logging.getLogger(__name__)
 
@@ -31,7 +30,7 @@
 CONCURRENCY = 50
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def get_root_url(use_proxy):
     """Get the current TASKCLUSTER_ROOT_URL.
 
@@ -106,7 +105,7 @@ def requests_retry_session(
     return session
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def get_session():
     return requests_retry_session(retries=5)
 
@@ -277,7 +276,7 @@ def get_task_url(task_id, use_proxy=False):
     return task_tmpl.format(task_id)
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def get_task_definition(task_id, use_proxy=False):
     response = _do_request(get_task_url(task_id, use_proxy))
     return response.json()
@@ -446,7 +445,7 @@ def list_task_group_incomplete_tasks(task_group_id):
             break
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def _get_deps(task_ids, use_proxy):
     upstream_tasks = {}
     for task_id in task_ids:

src/taskgraph/util/taskcluster.py

@@ -2,10 +2,10 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import functools
 from dataclasses import dataclass
 
 from .keyed_by import evaluate_keyed_by
-from .memoize import memoize
 
 
 @dataclass
@@ -29,7 +29,7 @@ def implementation(self):
 }
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def worker_type_implementation(graph_config, worker_type):
     """Get the worker implementation and OS for the given workerType, where the
     OS represents the host system, not the target OS, in the case of
@@ -46,7 +46,7 @@ def worker_type_implementation(graph_config, worker_type):
     return worker_config["implementation"], worker_config.get("os")
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def get_worker_type(graph_config, alias, level):
     """
     Get the worker type based, evaluating aliases from the graph config.