ci: reduce build time (#247)

CI build times were extremely ridiculous; now, they are only somewhat ridiculous.

* ci: optimize downloading clojure deps

Do the work from build.clj to avoid repeated JVM launch costs.

* ci: only install planck if job requires it

Planck can take a while to install on CI Linux, so only install it if necessary.

* lint fix: unsorted namespace

* ci yaml fixes

* ci: fix oopsie in libs tests

typo resulted in running ALL libs test for each invidual lib job

* ci: only npm if required by unit test

Sometimes `npm ci` is zippy on ci, sometimes not.
Skip it if not needed by particular unit test.

* ci: run against all clj versions in a single job

Clojure tests now hit all supported versions of Clojure in a single job.
This reduces number of GitHub Actions jobs which will reduce job queuing
which will reduce overall ci build time.

Author: lread

.github/workflows/libs-test.yml

@@ -35,7 +35,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        lib: ${{ fromJSON(needs.enumerate-libs.outputs.libs) }}
+        include: ${{ fromJSON(needs.enumerate-libs.outputs.libs) }}
+
+    name: ${{ matrix.lib-name }}
 
     steps:
       - name: Checkout
@@ -46,11 +48,12 @@ jobs:
       - name: Setup
         uses: ./.github/workflows/shared-setup
         with:
-          clj-cache-prefix: clj-libs-deps-${{ matrix.lib }}
+          clj-cache-prefix: clj-libs-deps-${{ matrix.lib-name }}
           clj-cache-hash-files: "'script/test_libs.clj','deps.edn','bb.edn'"
 
-      - name: Install planck
+      - name: Install Planck
         uses: ./.github/workflows/setup-planck
+        if: contains( matrix.requires, 'planck' )
 
       - name: Run Libs Tests
-        run: bb test-libs run ${{ matrix.lib }}
+        run: bb test-libs run ${{ matrix.lib-name }}

.github/workflows/unit-test.yml

@@ -57,17 +57,19 @@ jobs:
 
       - name: Install Planck
         uses: ./.github/workflows/setup-planck
-        if: matrix.os != 'windows'
+        if: contains( matrix.requires, 'planck' )
 
       - name: Node modules cache
         uses: actions/cache@v3
         with:
           path: ./node_modules
           key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
           restore-keys: ${{ runner.os }}-node-
+        if: contains( matrix.requires, 'npm' )
 
       - name: Install node packages
         run: npm ci
+        if: contains( matrix.requires, 'npm' )
 
       - name: Run Tests
         run: ${{ matrix.cmd }}

build.clj

@@ -1,6 +1,8 @@
 (ns build
   (:require [build-shared]
-            [clojure.tools.build.api :as b]))
+            [clojure.edn :as edn]
+            [clojure.tools.build.api :as b]
+            [clojure.tools.deps :as deps]))
 
 (def version (build-shared/lib-version))
 (def lib (build-shared/lib-artifact-name))
@@ -48,3 +50,18 @@
    {:installer :remote
     :artifact jar-file
     :pom-file (b/pom-path {:lib lib :class-dir class-dir})}))
+
+(defn download-deps
+  "Download all deps for all aliases"
+  [_]
+  (let [aliases (->> "deps.edn"
+                     slurp
+                     edn/read-string
+                     :aliases
+                     keys)]
+    ;; one at a time because aliases with :replace-deps will... well... you know.
+    (println "Bring down default deps")
+    (deps/create-basis {})
+    (doseq [a (sort aliases)]
+      (println "Bring down deps for alias" a)
+      (deps/create-basis {:aliases [a]}))))

script/ci_unit_tests.clj

@@ -27,8 +27,9 @@
            {:desc "import-vars" :cmd "bb apply-import-vars check" :oses all-oses :jdks ["8"]}
            {:desc "lint"        :cmd "bb lint"                    :oses all-oses :jdks ["8"]}
            ;; test-docs on default clojure version across all oses and jdks
-           {:desc "test-doc"          :cmd "bb test-doc"          :oses all-oses :jdks all-jdks}]
-          (for [version ["1.8" "1.9" "1.10" "1.11"]]
+           {:desc "test-doc"          :cmd "bb test-doc"          :oses all-oses :jdks all-jdks
+            :requires ["npm"]}]
+          (for [version ["all"]]
             {:desc (str "clj-" version)
              :cmd (str "bb test-clj --clojure-version " version)
              :oses all-oses
@@ -43,23 +44,26 @@
                         (when (:desc opt) (str "-" (:desc opt))))
              :cmd (str "bb test-cljs --env " (:param env) " --optimizations " (:param opt))
              :oses all-oses
-             :jdks ["8"]})
+             :jdks ["8"]
+             :requires ["npm"]})
           ;; shadow-cljs requires a min of jdk 11 so we'll test on that
           [{:desc "shadow-cljs"    :cmd "bb test-shadow-cljs" :oses all-oses :jdks ["11"]
             :skip-reason-fn (fn [{:keys [jdk]}] (when (< (parse-long jdk) 11)
-                                                  "jdk must be >= 11"))}]
+                                                  "jdk must be >= 11"))
+            :requires ["npm"]}]
           ;; planck does not run on windows, and I don't think it needs a jdk
           [{:desc "cljs-bootstrap" :cmd "bb test-cljs --env planck --optimizations none"
-            :oses ["macos" "ubuntu"] :jdks ["8"]}]))
+            :oses ["macos" "ubuntu"] :jdks ["8"] :requires ["planck"]}]))
 
 (defn- ci-test-matrix []
-  (for [{:keys [desc cmd oses jdks]} (test-tasks)
+  (for [{:keys [desc cmd oses jdks requires]} (test-tasks)
         os oses
         jdk jdks]
     {:desc (str desc " " os " jdk" jdk)
      :cmd cmd
      :os os
-     :jdk jdk}))
+     :jdk jdk
+     :requires (or requires [])}))
 
 (defn- local-test-list [local-os local-jdk]
   (for [{:keys [desc cmd oses skip-reason-fn]} (test-tasks)]
@@ -96,7 +100,7 @@ By default, will run all tests applicable to your current jdk and os.")
         (if (= "json" (get opts "--format"))
           (status/line :detail (json/generate-string matrix))
           (do
-            (status/line :detail (doric/table [:os :jdk :desc :cmd] matrix))
+            (status/line :detail (doric/table [:os :jdk :desc :cmd :requires] matrix))
             (status/line :detail "Total jobs found: %d" (count matrix)))))
       (let [cur-os (matrix-os)
             cur-jdk (jdk/version)

script/download_deps.clj

@@ -1,20 +1,9 @@
 (ns download-deps
-  (:require [babashka.tasks :as t]
-            [clojure.edn :as edn]
-            [lread.status-line :as status]))
+  (:require [babashka.tasks :as t]))
 
 ;; clojure has a -P command, but to bring down all deps we need to specify all aliases
 ;; bb deps will be brought down just from running bb (which assumedly is how this code is run)
 
 (defn -main [& _args]
-  (let [aliases (->> "deps.edn"
-                     slurp
-                     edn/read-string
-                     :aliases
-                     keys)]
-    ;; one at a time because aliases with :replace-deps will... well... you know.
-    (status/line :detail "Bring down default deps")
-    (t/clojure "-P")
-    (doseq [a (sort aliases)]
-      (status/line :detail "Bring down deps for alias: %s" a)
-      (t/clojure "-P" (str "-M" a)))))
+  ;; do all the work from build.clj to avoid repeated JVM launch costs
+  (t/clojure "-T:build download-deps"))

script/test_clj.clj

@@ -28,17 +28,21 @@
 (def args-usage "Valid args: [options]
 
 Options:
-  -v, --clojure-version VERSION  Test with Clojure [1.8, 1.9, 1.10, 1.11] [default: 1.8]
+  -v, --clojure-version VERSION  Test with Clojure [1.8, 1.9, 1.10, 1.11 all] [default: 1.8]
   --help                         Show this help")
 
 (defn -main [& args]
   (when-let [opts (main/doc-arg-opt args-usage args)]
     (let [clojure-version (get opts "--clojure-version")]
-      (if (not (some #{clojure-version} allowed-clojure-versions))
+
+      (if (not (some #{clojure-version} (conj allowed-clojure-versions "all")))
         (status/die 1 args-usage)
-        (do
-          (run-unit-tests clojure-version)
-          (run-isolated-tests clojure-version)))))
+        (let [clojure-versions (if (= "all" clojure-version)
+                                 allowed-clojure-versions
+                                 [clojure-version])]
+          (doseq [v clojure-versions]
+            (run-unit-tests v)
+            (run-isolated-tests v))))))
   nil)
 
 (main/when-invoked-as-script

script/test_libs.clj

@@ -441,6 +441,7 @@
             :test-cmds ["clojure -M:cljtest"
                         ;; disable zprint cljs tests for now, see https://github.com/planck-repl/planck/issues/1088
                         #_"clojure -M:cljs-runner"]
+            ;; :requires ["planck"] ;; re-enable when cljs tests are re-enabled
             :cleanup-fn zprint-cleanup}])
 
 (defn- header [text]
@@ -558,7 +559,11 @@ Specifying no lib-names selects all libraries.")
     (cond
       (get opts "list")
       (if (= "json" (get opts "--format"))
-        (status/line :detail (->> libs (map :name) json/generate-string))
+        (status/line :detail (->> libs
+                                  (map (fn [{:keys [name requires]}]
+                                         {:lib-name name
+                                          :requires (or requires [])}))
+                                  json/generate-string))
         (status/line :detail (str "available libs: " (string/join " " (map :name libs)))))
 
       :else