Optimizing CircleCI config to reduce build times (#594)

chrisrink10 · web-flow · commit cfe09bbeb977 · 2020-06-25T09:49:37.000-04:00
Circle builds were taking 20-40 minutes running PyPy on one node, which really discouraged me from breaking up PRs into smaller pieces since I'd have to wait often times nearly an hour just to merge in a smaller change.

This PR breaks up Circle builds into multiple different nodes. PyPy runs on three different workers using Circle's builtin `parallelism` and test-splitting features. Each of the CPython test suites run in their own node with MyPy/PyLint (to ensure we're not missing any type or linting errors across versions) and Safety (to ensure dependency versions installed on different versions do not have any known CVEs) checks. CI checks which do not vary by version are checked on Python 3.8, since it's faster than other versions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,64 +1,163 @@
-version: 2
-jobs:
-  test-cpython:
-    docker:
-      - image: chrisrink10/pyenv:3.6-3.7-3.8-0.0.1
-        user: pyenv
+version: 2.1
+
+commands:
+  setup_tests:
+    description: "Check out code, install Tox, and prepare the test environment."
+    parameters:
+      python_version:
+        description: "Required. Python version as `major.minor`."
+        type: string
+      cache_key_prefix:
+        description: "Required. Prefix used for the CircleCI cache key."
+        type: string
     steps:
       - checkout
+      - run: sudo chown -R circleci:circleci /usr/local/bin
+      - run: sudo chown -R circleci:circleci /usr/local/lib/python<< parameters.python_version >>/site-packages
       - restore_cache:
-          key: deps-{{ .Branch }}-{{ checksum "tox.ini" }}-{{ checksum "Pipfile.lock" }}
+          key: << parameters.cache_key_prefix >>1-{{ checksum "tox.ini" }}-{{ checksum "Pipfile.lock" }}
       - run:
           name: Install tox
           shell: /bin/bash -leo pipefail
           command: |
+            pip install -U pip
             pip install tox
+
+  teardown_tests:
+    description: "Store the cache for the current run."
+    parameters:
+      python_version:
+        description: "Required. Python version as `major.minor`."
+        type: string
+      cache_key_prefix:
+        description: "Required. Prefix used for the CircleCI cache key."
+        type: string
+    steps:
+      - save_cache:
+          key: << parameters.cache_key_prefix >>1-{{ checksum "tox.ini" }}-{{ checksum "Pipfile.lock" }}
+          paths:
+            - "/home/circleci/project/.tox"
+            - "/usr/local/bin"
+            - "/usr/local/lib/python<< parameters.python_version >>/site-packages"
+
+  run_cpython_tests:
+    description: "Install Tox and run tests."
+    parameters:
+      python_version:
+        description: "Required. Python version as `major.minor`."
+        type: string
+      tox_envs:
+        description: "Required. Set of Tox environments to run on this node."
+        type: string
+      tox_parallel:
+        description: "Optional. Number of parallel workers spawned by Tox."
+        type: integer
+        default: 2
+    steps:
+      - setup_tests:
+          python_version: << parameters.python_version >>
+          cache_key_prefix: py<< parameters.python_version >>-deps
       - run:
           name: Run Tests
           shell: /bin/bash -leo pipefail
           environment:
-            TOX_NUM_CORES: 2
             TOX_PARALLEL_NO_SPINNER: 1
             TOX_SHOW_OUTPUT: "True"
-            TOX_SKIP_ENV: pypy3
           command: |
-            tox -p $TOX_NUM_CORES
-      - save_cache:
-          key: deps9-{{ .Branch }}-{{ checksum "tox.ini" }}-{{ checksum "Pipfile.lock" }}
-          paths:
-            - "/home/pyenv/.tox"
-            - "/usr/local/bin"
-            - "/usr/local/lib/python3.6/site-packages"
+            tox -p << parameters.tox_parallel >> -e << parameters.tox_envs >>
+            mkdir coverage
+            mv .coverage.* "coverage/.coverage.py<< parameters.python_version >>"
+      - teardown_tests:
+          python_version: << parameters.python_version >>
+          cache_key_prefix: py<< parameters.python_version >>-deps
+      - store_artifacts:
+          path: coverage
       - store_test_results:
           path: junit
 
+jobs:
+  test-cpython-36:
+    docker:
+      - image: circleci/python:3.6-buster
+    steps:
+      - run_cpython_tests:
+          python_version: "3.6"
+          tox_envs: py36,py36-mypy,py36-lint,safety
+
+  test-cpython-37:
+    docker:
+      - image: circleci/python:3.7-buster
+    steps:
+      - run_cpython_tests:
+          python_version: "3.7"
+          tox_envs: py37,py37-mypy,py37-lint,safety
+
+  test-cpython-38:
+    docker:
+      - image: circleci/python:3.8-buster
+    steps:
+      - run_cpython_tests:
+          python_version: "3.8"
+          tox_envs: py38,py38-mypy,py38-lint,format,safety
+
+  report-coverage:
+    docker:
+      - image: circleci/python:3.8-buster
+    steps:
+      - setup_tests:
+          python_version: "3.8"
+          cache_key_prefix: report-coverage-deps
+      - run:
+          name: Report Coverage
+          command: |
+            # Fetch the build numbers for this Workflow UUID
+            RECENT_BUILDS_URL="https://circleci.com/api/v1.1/project/github/basilisp-lang/basilisp/tree/$CIRCLE_BRANCH"
+            BUILD_NUMS=$(curl -H "Circle-Token: $CIRCLECI_API_TOKEN" "$RECENT_BUILDS_URL" | \
+              jq -r "map(select(.workflows.workflow_id == \"$CIRCLE_WORKFLOW_ID\")) | map(.build_num) | .[]")
+            echo "CircleCI build URL: $RECENT_BUILDS_URL"
+            echo "CircleCI build numbers: $(echo "$BUILD_NUMS" | tr '\n' ' ')"
+
+            # Fetch all of the artifacts for the build numbers
+            for build_num in $BUILD_NUMS
+            do
+              ARTIFACT_META_URL="https://circleci.com/api/v1.1/project/github/basilisp-lang/basilisp/$build_num/artifacts"
+              echo "Fetching artifacts for CircleCI build from: $ARTIFACT_META_URL"
+              ARTIFACT_URLS=$(curl -H "Circle-Token: $CIRCLECI_API_TOKEN" "$ARTIFACT_META_URL" | jq -r '.[].url')
+              if [ -n "$ARTIFACT_URLS" ]; then
+                echo "Found artifact URLs: $(echo "$ARTIFACT_URLS" | tr '\n' ' ')"
+                curl -L --remote-name-all $ARTIFACT_URLS
+              fi
+            done
+            tox -v -e coverage
+      - teardown_tests:
+          python_version: "3.8"
+          cache_key_prefix: report-coverage-deps
+
   test-pypy:
     docker:
       - image: pypy:3.6-7-slim-buster
     parallelism: 3
     steps:
       - checkout
       - restore_cache:
-          key: deps-{{ .Branch }}-{{ checksum "tox.ini" }}-{{ checksum "Pipfile.lock" }}
+          key: pypy-deps2-{{ checksum "tox.ini" }}-{{ checksum "Pipfile.lock" }}
       - run:
           name: Install tox
           command: |
+            pip install -U pip
             pip install tox
       - run:
           name: Run Tests
           command: |
-            # Split tests by timing and print out the test files on each node
             CCI_NODE_TESTS=$(circleci tests glob "tests/**/*_test.py" "tests/**/test_*.py" | circleci tests split --split-by=timings)
             printf "Test files:\n"
             echo "$CCI_NODE_TESTS"
             printf "\n"
-
-            # Run the tests on the subset defined for this node
             tox -e pypy3 -- $CCI_NODE_TESTS
       - save_cache:
-          key: deps9-{{ .Branch }}-{{ checksum "tox.ini" }}-{{ checksum "Pipfile.lock" }}
+          key: pypy-deps2-{{ checksum "tox.ini" }}-{{ checksum "Pipfile.lock" }}
           paths:
-            - "/home/pyenv/.tox"
+            - "/root/project/.tox"
             - "/usr/local/bin"
             - "/usr/local/lib/python3.6/site-packages"
       - store_test_results:
@@ -68,5 +167,12 @@ workflows:
   version: 2
   test:
     jobs:
-      - test-cpython
+      - test-cpython-36
+      - test-cpython-37
+      - test-cpython-38
       - test-pypy
+      - report-coverage:
+          requires:
+            - test-cpython-36
+            - test-cpython-37
+            - test-cpython-38
diff --git a/src/basilisp/_pyast.py b/src/basilisp/_pyast.py
@@ -64,7 +64,7 @@
     LtE,
     MatMult,
     Mod,
-    Module,
+    Module as _Module,
     Mult,
     Name,
     NameConstant,
@@ -102,7 +102,7 @@
     YieldFrom,
     alias,
     arg,
-    arguments,
+    arguments as _arguments,
     boolop,
     cmpop,
     comprehension,
@@ -127,7 +127,24 @@
     walk,
     withitem,
 )
-from functools import partial
+
+if sys.version_info >= (3, 8):
+
+    class Module(_Module):
+        def __init__(self, *args, **kwargs):
+            kwargs["type_ignores"] = []
+            super().__init__(*args, **kwargs)
+
+    class arguments(_arguments):
+        def __init__(self, *args, **kwargs):
+            kwargs["posonlyargs"] = []
+            super().__init__(*args, **kwargs)
+
+
+else:
+    Module = _Module
+    arguments = _arguments
+
 
 __all__ = [
     "AST",
@@ -257,7 +274,3 @@
     "walk",
     "withitem",
 ]
-
-if sys.version_info >= (3, 8):  # pragma: no cover
-    Module = partial(Module, type_ignores=[])
-    arguments = partial(arguments, posonlyargs=[])
diff --git a/src/basilisp/lang/compiler/analyzer.py b/src/basilisp/lang/compiler/analyzer.py
@@ -3382,7 +3382,7 @@ def _const_node(ctx: AnalyzerContext, form: ReaderForm) -> Const:
 
 
 @_with_loc  # noqa: MC0001
-def _analyze_form(  # pylint: disable=too-many-branches
+def _analyze_form(  # pylint: disable=too-many-branches  # noqa: MC0001
     ctx: AnalyzerContext, form: Union[ReaderForm, ISeq]
 ) -> Node:
     if isinstance(form, (llist.List, ISeq)):
diff --git a/src/basilisp/lang/futures.py b/src/basilisp/lang/futures.py
@@ -64,6 +64,7 @@ class ProcessPoolExecutor(_ProcessPoolExecutor):  # pragma: no cover
     def __init__(self, max_workers: Optional[int] = None):
         super().__init__(max_workers=max_workers)
 
+    # pylint: disable=arguments-differ
     def submit(  # type: ignore
         self, fn: Callable[..., T], *args, **kwargs
     ) -> "Future[T]":
@@ -78,6 +79,7 @@ def __init__(
     ):
         super().__init__(max_workers=max_workers, thread_name_prefix=thread_name_prefix)
 
+    # pylint: disable=arguments-differ
     def submit(  # type: ignore
         self, fn: Callable[..., T], *args, **kwargs
     ) -> "Future[T]":
diff --git a/src/basilisp/lang/interfaces.py b/src/basilisp/lang/interfaces.py
@@ -29,6 +29,7 @@ def deref(self) -> Optional[T]:
 class IBlockingDeref(IDeref[T]):
     __slots__ = ()
 
+    # pylint: disable=arguments-differ
     @abstractmethod
     def deref(
         self, timeout: Optional[float] = None, timeout_val: Optional[T] = None
diff --git a/src/basilisp/lang/reader.py b/src/basilisp/lang/reader.py
@@ -110,9 +110,8 @@ class Comment:
 RawReaderForm = Union[ReaderForm, "ReaderConditional"]
 
 
-@attr.s(  # pylint:disable=redefined-builtin
-    auto_attribs=True, repr=False, slots=True, str=False
-)
+# pylint:disable=redefined-builtin
+@attr.s(auto_attribs=True, repr=False, slots=True, str=False)
 class SyntaxError(Exception):
     message: str
     line: Optional[int] = None
diff --git a/src/basilisp/lang/vector.py b/src/basilisp/lang/vector.py
@@ -25,7 +25,7 @@ class TransientVector(ITransientVector[T]):
     __slots__ = ("_inner",)
 
     def __init__(self, wrapped: "PVectorEvolver[T]") -> None:
-        self._inner = wrapped
+        self._inner = wrapped  # pylint: disable=assigning-non-slot
 
     def __bool__(self):
         return True
diff --git a/tox.ini b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = py36,py37,py38,pypy3,coverage,mypy,format,lint,safety
+envlist = py36,py37,py38,pypy3,coverage,py{36,37,38}-mypy,py{36,37,38}-lint,format,safety
 
 [testenv]
 parallel_show_output = {env:TOX_SHOW_OUTPUT:true}
@@ -64,12 +64,12 @@ commands =
     isort --check-only
     black --check .
 
-[testenv:mypy]
+[testenv:py{36,37,38}-mypy]
 deps = mypy
 commands =
     mypy --config-file={toxinidir}/mypy.ini --show-error-codes src/basilisp
 
-[testenv:lint]
+[testenv:py{36,37,38}-lint]
 deps = prospector==1.2.0
 commands =
     prospector --profile-path={toxinidir}
