From 9c76d0c0957f92724406d551d10dfcaef25d3aaa Mon Sep 17 00:00:00 2001 From: Bozhidar Batsov Date: Tue, 1 Jan 2019 10:50:13 +0200 Subject: [PATCH 01/21] Update REPL-y to 0.4.3 --- CHANGES.md | 6 ++++++ boot/worker/build.boot | 2 +- boot/worker/project.clj | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f512eb56..51daa9d4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,11 @@ # Changes +## 2.8.3 + +#### Improved + +- Updated REPL-y to 0.4.3. + ## 2.8.2 #### Fixed diff --git a/boot/worker/build.boot b/boot/worker/build.boot index 38627bb2..1215f6b3 100644 --- a/boot/worker/build.boot +++ b/boot/worker/build.boot @@ -2,7 +2,7 @@ :source-paths #{"src" "test"} :dependencies '[[net.cgrand/parsley "0.9.3" :exclusions [org.clojure/clojure]] [mvxcvi/puget "1.0.1"] - [reply "0.4.1"] + [reply "0.4.3"] [cheshire "5.3.1"] [clj-jgit "0.8.0"] [clj-yaml "0.4.0"] diff --git a/boot/worker/project.clj b/boot/worker/project.clj index 454c9586..e8d9fb0c 100644 --- a/boot/worker/project.clj +++ b/boot/worker/project.clj @@ -23,7 +23,7 @@ ;; see https://github.com/boot-clj/boot/issues/82 [net.cgrand/parsley "0.9.3" :exclusions [org.clojure/clojure]] [mvxcvi/puget "1.0.1"] - [reply "0.4.1"] + [reply "0.4.3"] [cheshire "5.6.0"] [clj-jgit "0.8.0"] [clj-yaml "0.4.0"] From ec9195323fb65d97c9b0c13d5b796ae106f277a9 Mon Sep 17 00:00:00 2001 From: Bozhidar Batsov Date: Tue, 1 Jan 2019 10:50:51 +0200 Subject: [PATCH 02/21] Update nREPL to 0.5.3 --- CHANGES.md | 1 + boot/pod/src/boot/repl.clj | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 51daa9d4..c6a032c8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,7 @@ #### Improved - Updated REPL-y to 0.4.3. +- Updated nREPL to 0.5.3. ## 2.8.2 diff --git a/boot/pod/src/boot/repl.clj b/boot/pod/src/boot/repl.clj index 146f89c9..c1afdd4e 100644 --- a/boot/pod/src/boot/repl.clj +++ b/boot/pod/src/boot/repl.clj @@ -8,7 +8,7 @@ [boot.from.io.aviso.exception :refer [*fonts*]])) (def ^:dynamic *default-dependencies* - (atom '[[nrepl/nrepl "0.4.4" :exclusions [[org.clojure/clojure]]]])) + (atom '[[nrepl/nrepl "0.5.3" :exclusions [[org.clojure/clojure]]]])) (defn ^:private disable-exception-colors [handler] From 00bfcaf32b8296db707247f82a105b700239f8e1 Mon Sep 17 00:00:00 2001 From: Bozhidar Batsov Date: Tue, 1 Jan 2019 10:54:47 +0200 Subject: [PATCH 03/21] Remove redundant exclusion nREPL doesn't have a hard Clojure dependency these days. --- boot/pod/src/boot/repl.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boot/pod/src/boot/repl.clj b/boot/pod/src/boot/repl.clj index c1afdd4e..94bb4b89 100644 --- a/boot/pod/src/boot/repl.clj +++ b/boot/pod/src/boot/repl.clj @@ -8,7 +8,7 @@ [boot.from.io.aviso.exception :refer [*fonts*]])) (def ^:dynamic *default-dependencies* - (atom '[[nrepl/nrepl "0.5.3" :exclusions [[org.clojure/clojure]]]])) + (atom '[[nrepl/nrepl "0.5.3"]])) (defn ^:private disable-exception-colors [handler] From ca03c0574159b4bad82bd83483f14138c79e0084 Mon Sep 17 00:00:00 2001 From: Patrik Sundberg Date: Wed, 13 Feb 2019 22:59:09 +0000 Subject: [PATCH 04/21] Upgrade dependencies to fix test failures on JDK 11. The toArray method is now overloaded and requires type hints, and through transitive depencies the boot-alt-test dependency dragged in an old version of core.rrb-vector breakin the build on jdk11. --- boot/core/build.boot | 8 ++++---- boot/worker/build.boot | 18 +++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/boot/core/build.boot b/boot/core/build.boot index 65c7dc8a..ec80403f 100644 --- a/boot/core/build.boot +++ b/boot/core/build.boot @@ -1,12 +1,12 @@ (set-env! :source-paths #{"src" "test"} - :dependencies '[[org.clojure/tools.reader "1.0.0-alpha2"] - [metosin/boot-alt-test "0.3.2" :scope "test"]]) + :dependencies '[[org.clojure/tools.reader "1.3.2" :exclusions [org.clojure/clojure]] + [metosin/bat-test "0.4.2" :scope "test"]]) (ns-unmap 'boot.user 'test) (require '[boot.test :refer [runtests test-report test-exit]] - '[metosin.boot-alt-test :refer [alt-test]] + '[metosin.bat-test :refer [bat-test]] 'boot.task.built-in-test 'boot.test-test) @@ -19,7 +19,7 @@ (test-exit))) (deftask unit-test [] - (alt-test :test-matcher #"boot\.cli-test")) + (bat-test :test-matcher #"boot\.cli-test")) (deftask test [] (comp diff --git a/boot/worker/build.boot b/boot/worker/build.boot index 38627bb2..dc757663 100644 --- a/boot/worker/build.boot +++ b/boot/worker/build.boot @@ -1,23 +1,23 @@ (set-env! :source-paths #{"src" "test"} :dependencies '[[net.cgrand/parsley "0.9.3" :exclusions [org.clojure/clojure]] - [mvxcvi/puget "1.0.1"] - [reply "0.4.1"] - [cheshire "5.3.1"] - [clj-jgit "0.8.0"] + [mvxcvi/puget "1.1.0"] + [reply "0.4.3"] + [cheshire "5.8.1"] + [clj-jgit "0.8.10"] [clj-yaml "0.4.0"] [javazoom/jlayer "1.0.1"] - [net.java.dev.jna/jna "4.1.0"] + [net.java.dev.jna/jna "5.2.0"] [alandipert/desiderata "1.0.2"] [org.clojure/data.xml "0.0.8"] - [org.clojure/data.zip "0.1.1"] + [org.clojure/data.zip "0.1.2"] [org.clojure/tools.namespace "0.2.11"] - [metosin/boot-alt-test "0.3.2" :scope "test"]]) + [metosin/bat-test "0.4.2" :scope "test"]]) (ns-unmap 'boot.user 'test) -(require '[metosin.boot-alt-test :refer [alt-test]]) +(require '[metosin.bat-test :refer [bat-test]]) (import boot.App) @@ -25,4 +25,4 @@ (comp (with-pass-thru [fs] (boot.util/info "Testing against version %s\n" (App/config "BOOT_VERSION"))) - (alt-test))) + (bat-test))) From 096b892abf1c7e7005a62c37a261797b06bb0b37 Mon Sep 17 00:00:00 2001 From: Bozhidar Batsov Date: Tue, 19 Mar 2019 16:48:58 +0200 Subject: [PATCH 05/21] Update nREPL to 0.6 --- CHANGES.md | 2 +- boot/pod/src/boot/repl.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c6a032c8..4db3d2d7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,7 +5,7 @@ #### Improved - Updated REPL-y to 0.4.3. -- Updated nREPL to 0.5.3. +- Updated nREPL to 0.6.0. ## 2.8.2 diff --git a/boot/pod/src/boot/repl.clj b/boot/pod/src/boot/repl.clj index 94bb4b89..4abfb624 100644 --- a/boot/pod/src/boot/repl.clj +++ b/boot/pod/src/boot/repl.clj @@ -8,7 +8,7 @@ [boot.from.io.aviso.exception :refer [*fonts*]])) (def ^:dynamic *default-dependencies* - (atom '[[nrepl/nrepl "0.5.3"]])) + (atom '[[nrepl/nrepl "0.6.0"]])) (defn ^:private disable-exception-colors [handler] From 7bc8b5600ecf16ed5b1e0863328fd719cd5bd936 Mon Sep 17 00:00:00 2001 From: Patrik Sundberg Date: Wed, 20 Mar 2019 10:20:38 +0000 Subject: [PATCH 06/21] upgrade puget to latest point release --- boot/worker/build.boot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boot/worker/build.boot b/boot/worker/build.boot index dc757663..85c9242e 100644 --- a/boot/worker/build.boot +++ b/boot/worker/build.boot @@ -1,7 +1,7 @@ (set-env! :source-paths #{"src" "test"} :dependencies '[[net.cgrand/parsley "0.9.3" :exclusions [org.clojure/clojure]] - [mvxcvi/puget "1.1.0"] + [mvxcvi/puget "1.1.1"] [reply "0.4.3"] [cheshire "5.8.1"] [clj-jgit "0.8.10"] From 9cd53ac649a75f44caa26e1805bae382c11340ab Mon Sep 17 00:00:00 2001 From: Matthew Ratzke Date: Fri, 12 Apr 2019 15:32:29 -0600 Subject: [PATCH 07/21] update java to 1.8 --- boot/worker/project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boot/worker/project.clj b/boot/worker/project.clj index e8d9fb0c..9eaee05d 100644 --- a/boot/worker/project.clj +++ b/boot/worker/project.clj @@ -14,7 +14,7 @@ :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} :java-source-paths ["third_party/barbarywatchservice/src"] - :javac-options ["-target" "1.7" "-source" "1.7"] + :javac-options ["-target" "1.8" "-source" "1.8"] :dependencies [[org.clojure/clojure "1.6.0" :scope "provided"] [boot/base ~version :scope "provided"] [boot/aether ~version] From 55e4b552f475b6ef434a523a6a3580cfcfd603e9 Mon Sep 17 00:00:00 2001 From: Matthew Ratzke Date: Fri, 12 Apr 2019 15:47:46 -0600 Subject: [PATCH 08/21] update boot dependencies --- boot/aether/project.clj | 5 +++-- boot/core/project.clj | 8 ++++---- boot/pod/project.clj | 9 +++++---- boot/worker/project.clj | 13 +++++++------ 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/boot/aether/project.clj b/boot/aether/project.clj index 1c6e2a30..87af24df 100644 --- a/boot/aether/project.clj +++ b/boot/aether/project.clj @@ -13,9 +13,10 @@ :repositories [["clojars" {:url "https://clojars.org/repo" :creds :gpg :sign-releases false}]] :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} + :plugins [[lein-ancient "0.6.15"]] :dependencies [[org.clojure/clojure "1.6.0" :scope "compile"] [boot/base ~version :scope "provided"] [boot/pod ~version :scope "compile"] - [com.cemerick/pomegranate "1.0.0" :scope "compile"] - [org.apache.maven.wagon/wagon-http "2.12" :scope "compile" + [com.cemerick/pomegranate "1.1.0" :scope "compile"] + [org.apache.maven.wagon/wagon-http "3.3.2" :scope "compile" :exclusions [org.apache.maven.wagon/wagon-provider-api]]]) diff --git a/boot/core/project.clj b/boot/core/project.clj index 4b22347c..48e7a2df 100644 --- a/boot/core/project.clj +++ b/boot/core/project.clj @@ -13,7 +13,7 @@ :repositories [["clojars" {:url "https://clojars.org/repo" :creds :gpg :sign-releases false}]] :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} - :dependencies [[org.clojure/clojure "1.6.0" :scope "provided"] - [boot/base ~version :scope "provided"] - [boot/pod ~version :scope "compile"]]) - + :plugins [[lein-ancient "0.6.15"]] + :dependencies [[org.clojure/clojure "1.6.0" :scope "provided"] + [boot/base ~version :scope "provided"] + [boot/pod ~version :scope "compile"]]) diff --git a/boot/pod/project.clj b/boot/pod/project.clj index b8601db2..227943d0 100644 --- a/boot/pod/project.clj +++ b/boot/pod/project.clj @@ -15,7 +15,8 @@ ["sonatype-snaps" {:url "https://oss.sonatype.org/content/repositories/snapshots"}]] :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} - :dependencies [[boot/base ~version :scope "provided"] - [org.clojure/clojure "1.6.0" :scope "provided"] - [org.tcrawley/dynapath "1.0.0" :scope "compile"] - [org.projectodd.shimdandy/shimdandy-impl "1.2.0" :scope "compile"]]) + :plugins [[lein-ancient "0.6.15"]] + :dependencies [[boot/base ~version :scope "provided"] + [org.clojure/clojure "1.6.0" :scope "provided"] + [org.tcrawley/dynapath "1.0.0" :scope "compile"] + [org.projectodd.shimdandy/shimdandy-impl "1.2.1" :scope "compile"]]) diff --git a/boot/worker/project.clj b/boot/worker/project.clj index 9eaee05d..5f409994 100644 --- a/boot/worker/project.clj +++ b/boot/worker/project.clj @@ -15,21 +15,22 @@ :url "http://www.eclipse.org/legal/epl-v10.html"} :java-source-paths ["third_party/barbarywatchservice/src"] :javac-options ["-target" "1.8" "-source" "1.8"] + :plugins [[lein-ancient "0.6.15"]] :dependencies [[org.clojure/clojure "1.6.0" :scope "provided"] [boot/base ~version :scope "provided"] [boot/aether ~version] ;; Suppress warnings from SLF4J via pomegranate via aether - [org.slf4j/slf4j-nop "1.7.22"] + [org.slf4j/slf4j-nop "1.7.26"] ;; see https://github.com/boot-clj/boot/issues/82 [net.cgrand/parsley "0.9.3" :exclusions [org.clojure/clojure]] - [mvxcvi/puget "1.0.1"] + [mvxcvi/puget "1.1.2"] [reply "0.4.3"] - [cheshire "5.6.0"] - [clj-jgit "0.8.0"] + [cheshire "5.8.1"] + [clj-jgit "0.8.10"] [clj-yaml "0.4.0"] [javazoom/jlayer "1.0.1"] - [net.java.dev.jna/jna "4.1.0"] + [net.java.dev.jna/jna "5.2.0"] [alandipert/desiderata "1.0.2"] [org.clojure/data.xml "0.0.8"] - [org.clojure/data.zip "0.1.1"] + [org.clojure/data.zip "0.1.3"] [org.clojure/tools.namespace "0.2.11"]]) From fb7f5c826c2cdfa0a6e9f7c6e3e7a7a7357c1ea7 Mon Sep 17 00:00:00 2001 From: Matthew Ratzke Date: Fri, 12 Apr 2019 15:47:50 -0600 Subject: [PATCH 09/21] Update version.properties --- version.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.properties b/version.properties index ece21668..9db130a2 100644 --- a/version.properties +++ b/version.properties @@ -1 +1 @@ -version=2.8.2 +version=2.8.3 From 519fe7dd0194c2b6ec3efa412ca7d8f90d104dd7 Mon Sep 17 00:00:00 2001 From: Matthew Ratzke Date: Fri, 12 Apr 2019 15:53:22 -0600 Subject: [PATCH 10/21] Update Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ee6b9c52..592d1277 100644 --- a/Makefile +++ b/Makefile @@ -82,8 +82,8 @@ install: .installed .deployed: .installed @echo -e "\033[0;33m<< Java version: $(java_version) >>\033[0m" - @[ "$(java_version)" == "1.7" ] \ - || (echo -e "\033[0;31mYou must build with Java version 1.7 only.\033[0m" && false) + @[ "$(java_version)" == "1.8" ] \ + || (echo -e "\033[0;31mYou must build with Java version 1.8 only.\033[0m" && false) (cd boot/base && lein deploy clojars boot/base $(version) target/base-$(version).jar pom.xml) (cd boot/pod && lein deploy clojars) (cd boot/aether && lein deploy clojars) From 4d0d807e80f99595cb43377ca0a49a64b84a1f3e Mon Sep 17 00:00:00 2001 From: Matthew Ratzke Date: Fri, 12 Apr 2019 15:55:55 -0600 Subject: [PATCH 11/21] remove java version check --- Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Makefile b/Makefile index 592d1277..362935c8 100644 --- a/Makefile +++ b/Makefile @@ -81,9 +81,6 @@ $(baseuber): boot/base/pom.xml $(shell find boot/base/src/main) install: .installed .deployed: .installed - @echo -e "\033[0;33m<< Java version: $(java_version) >>\033[0m" - @[ "$(java_version)" == "1.8" ] \ - || (echo -e "\033[0;31mYou must build with Java version 1.8 only.\033[0m" && false) (cd boot/base && lein deploy clojars boot/base $(version) target/base-$(version).jar pom.xml) (cd boot/pod && lein deploy clojars) (cd boot/aether && lein deploy clojars) From dde4659e65e611e830ef5741c3b15f65d7889674 Mon Sep 17 00:00:00 2001 From: Matthew Ratzke Date: Sat, 13 Apr 2019 15:47:29 -0600 Subject: [PATCH 12/21] remove aot from projects --- boot/aether/project.clj | 1 - boot/core/project.clj | 1 - boot/pod/project.clj | 1 - boot/worker/project.clj | 1 - 4 files changed, 4 deletions(-) diff --git a/boot/aether/project.clj b/boot/aether/project.clj index 87af24df..48caa071 100644 --- a/boot/aether/project.clj +++ b/boot/aether/project.clj @@ -5,7 +5,6 @@ (.getProperty "version"))) (defproject boot/aether version - :aot :all :jar-exclusions [#"^clojure/core/"] :description "Boot aether module–performs maven dependency resolution." :url "https://github.com/boot-clj/boot" diff --git a/boot/core/project.clj b/boot/core/project.clj index 48e7a2df..72fd7e7f 100644 --- a/boot/core/project.clj +++ b/boot/core/project.clj @@ -5,7 +5,6 @@ (.getProperty "version"))) (defproject boot/core version - :aot :all :jar-exclusions [#"^clojure/core/"] :description "Core boot module–boot scripts run in this pod." :url "https://github.com/boot-clj/boot" diff --git a/boot/pod/project.clj b/boot/pod/project.clj index 227943d0..0a2d05bf 100644 --- a/boot/pod/project.clj +++ b/boot/pod/project.clj @@ -5,7 +5,6 @@ (.getProperty "version"))) (defproject boot/pod version - :aot [#"^(?!boot\.repl-server).*$"] :jar-exclusions [#"^clojure/core/"] :description "Boot pod module–this is included with all pods." :url "https://github.com/boot-clj/boot" diff --git a/boot/worker/project.clj b/boot/worker/project.clj index 5f409994..98576b76 100644 --- a/boot/worker/project.clj +++ b/boot/worker/project.clj @@ -5,7 +5,6 @@ (.getProperty "version"))) (defproject boot/worker version - :aot :all :jar-exclusions [#"^clojure/core/"] :description "Boot worker module–this is the worker pod for built-in tasks." :url "https://github.com/boot-clj/boot" From da634df84272cade6170c87e719226c4ccb89931 Mon Sep 17 00:00:00 2001 From: Matthew Ratzke Date: Sat, 13 Apr 2019 16:10:55 -0600 Subject: [PATCH 13/21] restore pod aot --- boot/pod/project.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/boot/pod/project.clj b/boot/pod/project.clj index 0a2d05bf..227943d0 100644 --- a/boot/pod/project.clj +++ b/boot/pod/project.clj @@ -5,6 +5,7 @@ (.getProperty "version"))) (defproject boot/pod version + :aot [#"^(?!boot\.repl-server).*$"] :jar-exclusions [#"^clojure/core/"] :description "Boot pod module–this is included with all pods." :url "https://github.com/boot-clj/boot" From 5d2fbb167979394da44c068b22eea3a261affff2 Mon Sep 17 00:00:00 2001 From: Matthew Ratzke Date: Sun, 14 Apr 2019 15:27:13 -0600 Subject: [PATCH 14/21] update boot version --- boot.properties | 1 + version.properties | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 boot.properties diff --git a/boot.properties b/boot.properties new file mode 100644 index 00000000..89264136 --- /dev/null +++ b/boot.properties @@ -0,0 +1 @@ +BOOT_VERSION=3.0.0-SNAPSHOT diff --git a/version.properties b/version.properties index 9db130a2..cd92d6b0 100644 --- a/version.properties +++ b/version.properties @@ -1 +1 @@ -version=2.8.3 +version=3.0.0-SNAPSHOT From 0058c345590eb94bb8a9e6d27e68d617d3205e07 Mon Sep 17 00:00:00 2001 From: Matthew Ratzke Date: Sun, 14 Apr 2019 15:36:40 -0600 Subject: [PATCH 15/21] fix formatting and tests --- boot/pod/src/boot/util.clj | 12 +++++------ boot/pod/test/boot/util_test.clj | 36 ++++++++++++++++---------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/boot/pod/src/boot/util.clj b/boot/pod/src/boot/util.clj index 73dbe89a..47a2da02 100644 --- a/boot/pod/src/boot/util.clj +++ b/boot/pod/src/boot/util.clj @@ -48,7 +48,7 @@ (def ^:dynamic *verbosity* "Atom containing the verbosity level, 1 is lowest, 3 highest. Level 2 corresponds to the -v boot option, level 3 to -vv, etc. - + Levels: 1. Print INFO level messages or higher, colorize and prune stack traces @@ -173,7 +173,7 @@ (defmacro extends-protocol "Like extend-protocol but allows specifying multiple classes for each of the implementations: - + (extends-protocol IFoo clojure.lang.MapEntry ; <-- this is the difference, multiple clojure.lang.PersistentVector ; <-- classes per implementation @@ -232,7 +232,7 @@ (defmacro dotoseq "A cross between doto and doseq. For example: - + (-> (System/-err) (dotoseq [i (range 0 100)] (.printf \"i = %d\\n\" i)) @@ -252,7 +252,7 @@ is missing." [binding & body] (let [[ks m] [(butlast binding) (last binding)] - req-ks (set (map keyword ks)) ] + req-ks (set (map keyword ks))] `(if-let [dif-ks# (not-empty (set/difference ~req-ks (set (keys ~m))))] (throw (new AssertionError (apply format "missing key(s): %s" dif-ks#))) (let [{:keys ~ks} ~m] ~@body)))) @@ -339,9 +339,9 @@ A tree consists of a graph of nodes of the format [ ], where is a string and is a set of nodes (the children of this node). - + Example: - + (util/print-tree [[\"foo\" #{[\"bar\" #{[\"baz\"]}]}]] [\"--\" \"XX\"]) prints: diff --git a/boot/pod/test/boot/util_test.clj b/boot/pod/test/boot/util_test.clj index 90646476..ed43a681 100644 --- a/boot/pod/test/boot/util_test.clj +++ b/boot/pod/test/boot/util_test.clj @@ -4,13 +4,13 @@ [boot.util :as util :refer :all])) (deftest dep-mgt-functions - + (let [project 'com.example/project version "1.2.3" scope "test" exclusions [['com.example/excl1 :extension "jar"] 'com.example/excl2]] - + (testing "simple dep-as-map conversions" (are [input expected] (= expected (dep-as-map input)) @@ -23,22 +23,22 @@ {:project nil :version nil :scope "compile"} - + [project] {:project project :version nil :scope "compile"} - + [project version] {:project project :version version :scope "compile"} - + [project version :scope scope] {:project project :version version :scope scope} - + [project version :scope scope :exclusions exclusions] {:project project :version version @@ -58,28 +58,28 @@ {:project project :version nil :scope scope})) - - (testing "simple map-as-dep conversions" + + (testing "simple map-as-dep conversions" (are [input expected] (= expected (map-as-dep input)) {} [] - + {:project project :version nil :scope "compile"} [project] - + {:project project :version version :scope "compile"} [project version] - + {:project project :version version :scope scope} [project version :scope scope] - + {:project project :version version :exclusions exclusions} @@ -98,21 +98,21 @@ [project :scope scope])) (testing "roundtripping deps" - + (are [input] (= input (dep-as-map (map-as-dep input))) - + {:project project :version nil :scope "compile"} - + {:project project :version version :scope "compile"} - + {:project project :version version :scope scope} - + {:project project :version version :scope scope @@ -131,7 +131,7 @@ :scope scope})) (testing "check unusual arguments" - (is (thrown? Exception (dep-as-map {}))) + (is (map? (dep-as-map {}))) (is (= [] (map-as-dep nil))) (is (= [] (map-as-dep []))) (is (thrown? Exception (map-as-dep 3)))))) From 8ed925b050ccaa3a71babbd6c662f77c451c2735 Mon Sep 17 00:00:00 2001 From: Matthew Ratzke Date: Sat, 27 Apr 2019 12:31:45 -0600 Subject: [PATCH 16/21] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f78ff084..09ade3e8 100644 --- a/README.md +++ b/README.md @@ -329,6 +329,8 @@ it `build`. We'll modify `build.boot` such that it contains the following: [] (comp (pom) (jar) (install))) ``` +`NOTE: When using comp, all arguments must be functions - nil is not supported. +In this example we call each task middleware which returns the task function, these functions are composed into a new build task.` Now we should be able to see the `build` task listed among the available tasks in the output of `boot -h`, and we can run the task from the command line as we From b8e057fb5c942cbecea9068aa2d1b2adabc26f98 Mon Sep 17 00:00:00 2001 From: Matthew Ratzke Date: Sun, 28 Apr 2019 16:41:47 -0600 Subject: [PATCH 17/21] Update pod.clj --- boot/pod/src/boot/pod.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boot/pod/src/boot/pod.clj b/boot/pod/src/boot/pod.clj index b4f58f67..478f3f8b 100644 --- a/boot/pod/src/boot/pod.clj +++ b/boot/pod/src/boot/pod.clj @@ -465,7 +465,7 @@ ([expr] (let [{:keys [meta? expr]} (read-string expr)] (binding [*print-meta* meta?] - (pr-str (eval expr))))) + (pr-str (clojure.walk/prewalk identity (eval expr)))))) ([pod expr] (let [arg (pr-str {:meta? *print-meta* :expr expr}) ret (with-invoke-in pod (boot.pod/eval-in* arg))] From afd4d3f86bcb6f68c080949d16ad6fadbdd14d4a Mon Sep 17 00:00:00 2001 From: Matthew Ratzke Date: Mon, 29 Apr 2019 09:43:20 -0600 Subject: [PATCH 18/21] Update pod.clj --- boot/pod/src/boot/pod.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/boot/pod/src/boot/pod.clj b/boot/pod/src/boot/pod.clj index 478f3f8b..4ef1c970 100644 --- a/boot/pod/src/boot/pod.clj +++ b/boot/pod/src/boot/pod.clj @@ -2,6 +2,7 @@ (:require [clojure.set :as set] [clojure.string :as string] + [clojure.walk :as walk] [boot.util :as util] [boot.file :as file] [boot.xform :as xf] @@ -465,7 +466,7 @@ ([expr] (let [{:keys [meta? expr]} (read-string expr)] (binding [*print-meta* meta?] - (pr-str (clojure.walk/prewalk identity (eval expr)))))) + (pr-str (walk/prewalk identity (eval expr)))))) ([pod expr] (let [arg (pr-str {:meta? *print-meta* :expr expr}) ret (with-invoke-in pod (boot.pod/eval-in* arg))] From 7b9b7edffda9060595f86e8eb3757f7b473199b4 Mon Sep 17 00:00:00 2001 From: Dave Yarwood Date: Sat, 1 Jun 2019 14:04:31 -0400 Subject: [PATCH 19/21] add prepl-server task --- boot/core/src/boot/task/built_in.clj | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/boot/core/src/boot/task/built_in.clj b/boot/core/src/boot/task/built_in.clj index ed6c7108..75e85f72 100644 --- a/boot/core/src/boot/task/built_in.clj +++ b/boot/core/src/boot/task/built_in.clj @@ -531,6 +531,18 @@ (core/with-pass-thru [fs] @repl-soc))) +(core/deftask prepl-server + "Start a prepl server. + + This task is a thin wrapper around the `socket-server` task. See the + docstring for `socket-server` for details." + [b bind ADDR str "The address server listens on." + p port PORT int "The port to listen to."] + (socket-server + :bind bind + :accept 'clojure.core.server/io-prepl + :port port)) + (core/deftask pom "Create project pom.xml file. From dfc9faeffb5880e1bfa1a5a368080b2d2fe4ab5b Mon Sep 17 00:00:00 2001 From: Matthew Ratzke Date: Sat, 6 Jul 2019 20:19:06 -0600 Subject: [PATCH 20/21] wip fileset --- boot/fileset/boot.properties | 1 + boot/fileset/build.boot | 1 + boot/fileset/src/boot/file.clj | 312 +++++++++++++++++++++ boot/fileset/src/boot/filesystem.clj | 201 ++++++++++++++ boot/fileset/src/boot/tmpdir.clj | 388 +++++++++++++++++++++++++++ boot/fileset/version.properties | 1 + 6 files changed, 904 insertions(+) create mode 100644 boot/fileset/boot.properties create mode 100644 boot/fileset/build.boot create mode 100644 boot/fileset/src/boot/file.clj create mode 100644 boot/fileset/src/boot/filesystem.clj create mode 100644 boot/fileset/src/boot/tmpdir.clj create mode 100644 boot/fileset/version.properties diff --git a/boot/fileset/boot.properties b/boot/fileset/boot.properties new file mode 100644 index 00000000..19426b10 --- /dev/null +++ b/boot/fileset/boot.properties @@ -0,0 +1 @@ +BOOT_VERSION=2.8.3 diff --git a/boot/fileset/build.boot b/boot/fileset/build.boot new file mode 100644 index 00000000..9bdd45fd --- /dev/null +++ b/boot/fileset/build.boot @@ -0,0 +1 @@ +(set-env! :source-paths #{"src"}) diff --git a/boot/fileset/src/boot/file.clj b/boot/fileset/src/boot/file.clj new file mode 100644 index 00000000..64326326 --- /dev/null +++ b/boot/fileset/src/boot/file.clj @@ -0,0 +1,312 @@ +(ns boot.file + (:refer-clojure :exclude [list sync name file-seq]) + (:require + [clojure.java.io :as io] + [clojure.set :as set] + [clojure.data :as data] + [clojure.string :as str]) + (:import + [java.security MessageDigest] + [java.math BigInteger] + [java.io File] + [java.nio.file Files StandardCopyOption FileVisitOption] + [java.nio.file.attribute FileAttribute PosixFilePermissions])) + +(set! *warn-on-reflection* true) + +;; MD5 Digest ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +(defn md5 + "Converts string to MD5 digest." + [^String s] + (->> s + (.getBytes) + (.digest (MessageDigest/getInstance "MD5")) + (BigInteger. 1) + (format "%032x"))) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Boot Files API ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +(defn file? + "Check if argument is a file." + [^File f] + (.isFile (io/file f))) + +(defn directory? + "Check if argument is a directory." + [^File f] + (.isDirectory (io/file f))) + +(defn exists? + "Check if file/directory exists." + [^File f] + (.exists (io/file f))) + +(defn writeable? + "Check if file/directory is writeable." + [^File f] + (.canWrite (io/file f))) + +(defn path + "Return path of file/directory." + [^File f] + (.getPath (io/file f))) + +(defn name + "Return name of file/directory." + [^File f] + (.getName (io/file f))) + +(defn parent + "Return parent of file/directory." + [^File f] + (.getParentFile (io/file f))) + +(defn delete! + "Delete a file/directory." + [^File file] + (.delete (io/file file))) + +(defn list + "List files in a directory." + [^File dir] + (.listFiles (io/file dir))) + +(defn last-modified + "Get last modified data of file/directory." + [^File file] + (.getLastModifiedTime (io/file file))) + +(defn last-modified! + "Set last modified data of file/directory." + [^File file time] + (.setLastModifiedTime (io/file file) time)) + +(defn hard-link! + "Create a hard link from existing to link." + [existing link] + (let [^File link (path (io/file link)) + ^File existing (path (io/file existing))] + (Files/deleteIfExists link) + (Files/createLink link existing))) + +(defn sym-link! + "Create a symbolic link from target to link." + [target link] + (let [^File link (path (io/file link)) + ^File target (path (io/file target))] + (Files/deleteIfExists link) + (Files/createSymbolicLink link target))) + +(defn sym-link? + "Checks if file/directory is symbolic link." + [^File file] + (Files/isSymbolicLink (path (io/file file)))) + +(defn relative-path + "Returns a relativized path between base and target." + [base target] + (path (.relativize (path base) (path path)))) + +(defn file-seq + "Return a list of files from directory." + [directory & {:keys [symlinks] :or {symlinks true}}] + (when directory + (tree-seq + #(and (directory? %) + (or symlinks (not (sym-link? %)))) + #(seq (list %)) + (io/file directory)))) + +(defn empty-directory? + "Checks if directory is empty." + [directory] + (and (directory? f) (empty? (list f)))) + +(defn empty-directory! + "Delete all files within directory recursively." + [directory] + (when (exists? directory) + (let [file-seq #(file-seq % :symlinks false)] + (mapv delete! (-> directory file-seq rest reverse))))) + +(defn move + "Move a file from source to destination." + [^File src ^File dest] + (let [opts [StandardCopyOption/ATOMIC_MOVE + StandardCopyOption/REPLACE_EXISTING]] + (Files/move (path src) (path dest) (into-array StandardCopyOption opts)))) + +(defn replace-path + "Returns a file path by replacing source with destination directory." + [source destination file] + (path (io/file destination (relative-path (io/file source) (io/file file))))) + +(defn replace-paths + "Same as replace-path except operates on a collection of paths." + [source destination paths] + (map (partial replace-path source destination) paths)) + +(defn filter-files + "Returns a list containing only files from directory." + [directory] + (map path (filter file? (file-seq directory)))) + +(defn map-file + "Maps java.io.File to all paths." + [paths] + (map io/file paths)) + +(defn write-error + "Throws an error when unable to write to file/directory." + [target] + (let [path (path target) + msg (format "Can't write to file or directory (%s)." path)] + (throw (ex-info msg {:path path})))) + +(defn copy + "Copies a file from source to target directory, preserving last modified date." + [source target & opts] + (if-not (writeable? (io/file target)) + (write-error target) + (let [copy! (partial io/copy source)] + (if (:hard-link opts *hard-link*) + (hard-link! source target) + (let [date (last-modified source)] + (doto target io/make-parents copy! + (last-modified! date))))))) + +(defn copy-files + "Copy files from source directory to destination directory." + [source destination] + (when (exists? source) + (let [in-files (filter-files source) + out-files (replace-paths source destination in-files)] + (mapv copy (map-file in-files) (map-file out-files))))) + +(defn copy-atomically + "Copy file from source to destination, uses an atomic opperation." + [^File source ^File target] + (let [tmp (tmpfile (name target) nil (parent target))] + (binding [*hard-link* false] + (copy source tmp)) + (move tmp target))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(def ^:dynamic *ignore* nil) +(def ^:dynamic *hard-link* true) + +(def windows? (boot.App/isWindows)) + +(def tmpfile-permissions + (into-array FileAttribute + (if windows? + [] + [(PosixFilePermissions/asFileAttribute + (PosixFilePermissions/fromString "rw-------"))]))) + +(defn walk-file-tree + "Wrap java.nio.Files/walkFileTree to easily toggle symlink-following behavior." + [root visitor & {:keys [symlinks] :or {symlinks true}}] + (let [opts (if symlinks #{FileVisitOption/FOLLOW_LINKS} #{})] + (Files/walkFileTree root opts Integer/MAX_VALUE visitor))) + +(defmacro guard [& exprs] + `(try (do ~@exprs) (catch Throwable _#))) + +(def print-ex + (delay + (require 'boot.util) + (var-get (resolve 'boot.util/print-ex)))) + +(defn parent-seq + "Return sequence of this file and all it's parent directories" + [f] + (->> f io/file (iterate parent) (take-while identity))) + +(defn split-path + "Return a sequence of directories to this file path." + [p] + (->> p parent-seq reverse (map (fn [^File f] (name f))))) + +(defn parent? [parent child] + (contains? (set (parent-seq child)) parent)) + +(defn ^File tmpfile + ([prefix postfix] + (let [path (Files/createTempFile prefix postfix tmpfile-permissions)] + (doto (.toFile path) (.deleteOnExit)))) + ([prefix postfix ^File dir] + (let [path (Files/createTempFile (.toPath dir) prefix postfix tmpfile-permissions)] + (doto (.toFile path) (.deleteOnExit))))) + +(defn ^File tmpdir + ([prefix] + (io/file (Files/createTempDirectory prefix (into-array FileAttribute [])))) + ([^File dir prefix] + (io/file (Files/createTempDirectory (path dir) prefix (into-array FileAttribute []))))) + +(defn tree-for [& dirs] + (->> (for [dir dirs] + (let [path (-> (if (string? dir) dir (path ^File dir)) + ((fn [^String s] (.replaceAll s "/$" "")))) + snip (count (str path "/"))] + (->> (file-seq (io/file path)) + (reduce (fn [xs ^File f] + (if-not (.isFile f) + xs + (let [p (path f) + p' (subs p snip) + r #(re-find % p')] + (if (some r *ignore*) + xs + (-> (assoc-in xs [:file p'] f) + (assoc-in [:time p'] (last-modified f))))))) + {})))) + (reduce (partial merge-with into) {}))) + +(defn time-diff [before after] + ((fn [[b a]] [(set/difference b a) a]) + (->> (data/diff (:time before) (:time after)) (take 2) (map (comp set keys))))) + +(defmulti patch-cp? (fn [pred a b] pred)) +(defmethod patch-cp? :default [_ a b] true) +(defmethod patch-cp? :theirs [_ a b] true) +(defmethod patch-cp? :hash [_ a b] (not= (md5 a) (md5 b))) + +(defn patch [pred before after] + (let [[x cp] (time-diff before after) + rm (set/difference x cp)] + (concat + (for [x rm] [:rm x (get-in before [:file x])]) + (for [x cp :let [b (get-in before [:file x]) + a (get-in after [:file x])] + :when (patch-cp? pred a b)] + [:cp x a])))) + +(defn sync! [pred dest & srcs] + (let [before (tree-for dest) + after (apply tree-for srcs)] + (doseq [[op p x] (patch pred before after)] + (case op + :rm (delete! x) + :cp (copy-with-lastmod x (io/file dest p)))))) + +(defn watcher! [pred & dirs] + (let [state (atom nil)] + (fn [] + (let [state' (apply tree-for dirs) + patch' (patch pred @state state')] + (reset! state state') + patch')))) + +(defn match-filter? + [filters f] + (let [normalize #(if-not windows? % (str/replace % #"\\" "/"))] + ((apply some-fn (map (partial partial re-find) filters)) (normalize (path ^File f))))) + +(defn keep-filters? + [include exclude f] + (and + (or (empty? include) (match-filter? include f)) + (or (empty? exclude) (not (match-filter? exclude f))))) diff --git a/boot/fileset/src/boot/filesystem.clj b/boot/fileset/src/boot/filesystem.clj new file mode 100644 index 00000000..0f409df1 --- /dev/null +++ b/boot/fileset/src/boot/filesystem.clj @@ -0,0 +1,201 @@ +(ns boot.filesystem + (:require + [clojure.java.io :as io] + [clojure.set :as set] + [clojure.data :as data] + [clojure.string :as string] + [boot.filesystem.patch :as fsp] + [boot.file :as file] + [boot.util :as util :refer [with-let]]) + (:import + [java.net URI] + [java.io File] + [java.util.zip ZipEntry ZipOutputStream ZipException] + [java.util.jar JarEntry JarOutputStream Manifest Attributes$Name] + [java.nio.file.attribute FileAttribute FileTime PosixFilePermission + PosixFilePermissions] + [java.nio.file Path Files FileSystems StandardCopyOption StandardOpenOption + LinkOption SimpleFileVisitor FileVisitResult])) + +(set! *warn-on-reflection* true) + +(def continue FileVisitResult/CONTINUE) +(def skip-subtree FileVisitResult/SKIP_SUBTREE) + +(def ^"[Ljava.nio.file.LinkOption;" link-opts + (into-array LinkOption [])) + +(def ^"[Ljava.nio.file.attribute.FileAttribute;" tmp-attrs + (into-array FileAttribute [])) + +(def ^"[Ljava.nio.file.StandardCopyOption;" copy-opts + (into-array StandardCopyOption [StandardCopyOption/REPLACE_EXISTING])) + +(def ^"[Ljava.nio.file.StandardOpenOption;" open-opts + (into-array StandardOpenOption [StandardOpenOption/CREATE])) + +(def read-only + (PosixFilePermissions/fromString "r--r--r--")) + +(def windows? (boot.App/isWindows)) + +(defprotocol IToPath + (->path [x] "Returns a java.nio.file.Path for x.")) + +(extend-protocol IToPath + java.nio.file.Path + (->path [x] x) + + java.io.File + (->path [x] (.toPath x)) + + java.lang.String + (->path [x] (.toPath (io/file x))) + + java.nio.file.FileSystem + (->path [x] (first (.getRootDirectories x)))) + +(defn path->segs + [^Path path] + (->> path .iterator iterator-seq (map str))) + +(defn- ^Path segs->path + [^Path any-path-same-filesystem segs] + (let [segs-ary (into-array String (rest segs))] + (-> (.getFileSystem any-path-same-filesystem) + (.getPath (first segs) segs-ary)))) + +(defn- rel + [^Path root segs] + (.resolve root (segs->path root segs))) + +(defn mkjarfs + [^File jarfile & {:keys [create]}] + (when create (io/make-parents jarfile)) + (let [jaruri (->> jarfile .getCanonicalFile .toURI (str "jar:") URI/create) + ^java.util.Map opts {"create" (str (boolean create))}] + (FileSystems/newFileSystem jaruri opts))) + +(defn mkignores + [ignores] + (some->> (seq ignores) (map #(partial re-find %)) (apply some-fn))) + +(defn mkvisitor + [^Path root tree & {:keys [ignore]}] + (let [ign? (mkignores ignore)] + (proxy [SimpleFileVisitor] [] + (preVisitDirectory [path attr] + (let [p (.relativize root path)] + (try (if (and ign? (ign? (.toString p))) skip-subtree continue) + (catch java.nio.file.NoSuchFileException _ + (util/dbug* "Filesystem: file not found: %s\n" (.toString p)) + skip-subtree)))) + (visitFile [path attr] + (with-let [_ continue] + (let [p (.relativize root path) + s (path->segs p)] + (try (when-not (and ign? (ign? (.toString p))) + (->> (.toMillis (Files/getLastModifiedTime path link-opts)) + (hash-map :path s :file path :time) + (swap! tree assoc s))) + (catch java.nio.file.NoSuchFileException _ + (util/dbug* "Filesystem: file not found: %s\n" (.toString p)))))))))) + +(defrecord FileSystemTree [root tree]) + +(defn mktree + ([] (FileSystemTree. nil nil)) + ([root & {:keys [ignore]}] + (FileSystemTree. + root + @(with-let [tree (atom {})] + (file/walk-file-tree root (mkvisitor root tree :ignore ignore)))))) + +(defn merge-trees + [{tree1 :tree} {tree2 :tree}] + (FileSystemTree. (:root tree1) (merge tree1 tree2))) + +(defn tree-diff + [{t1 :tree :as before} {t2 :tree :as after}] + (let [reducer #(assoc %1 %2 (:time %3)) + [d1 d2] (map #(reduce-kv reducer {} %) [t1 t2]) + [x y _] (map (comp set keys) (data/diff d1 d2))] + {:adds (->> (set/difference y x) (select-keys t2) (assoc after :tree)) + :rems (->> (set/difference x y) (select-keys t1) (assoc after :tree)) + :chgs (->> (set/intersection x y) (select-keys t2) (assoc after :tree))})) + +(defn tree-patch + [before after link] + (let [->p (partial segs->path (:root before)) + writeop (if (= :all link) :link :write) + {:keys [adds rems chgs]} (tree-diff before after)] + (-> (->> rems :tree vals (map #(vector :delete (:path %)))) + (into (for [x (->> adds :tree (merge (:tree chgs)) vals)] + [writeop (:path x) (:file x) (:time x)]))))) + +(defmethod fsp/patch FileSystemTree + [before after link] + (tree-patch before after link)) + +(defmethod fsp/patch-result FileSystemTree + [before after] + (let [update-file #(assoc %1 :file (rel (:root before) %2)) + update-path #(-> after (get-in [:tree %]) (update-file %)) + update-tree (fn [xs k _] (assoc xs k (update-path k)))] + (assoc before :tree (reduce-kv update-tree {} (:tree after))))) + +(defn mkparents! + [^Path path] + (when-let [p (.getParent path)] + (Files/createDirectories p (into-array FileAttribute [])))) + +(defn touch! + [dest path time] + (let [dst (rel dest path)] + (util/dbug* "Filesystem: touching %s...\n" (string/join "/" path)) + (Files/setLastModifiedTime dst (FileTime/fromMillis time)))) + +(defn copy! + [^Path dest path ^Path src time & {:keys [mode]}] + (let [dst (doto (rel dest path) mkparents!)] + (util/dbug* "Filesystem: copying %s...\n" (string/join "/" path)) + (try (Files/copy ^Path src ^Path dst copy-opts) + (Files/setLastModifiedTime dst (FileTime/fromMillis time)) + (when (and mode (not windows?)) (Files/setPosixFilePermissions dst mode)) + (catch java.nio.file.NoSuchFileException ex + (util/dbug* "Filesystem: %s\n", (str ex)))))) + +(defn link! + [dest path src & {:keys [mode]}] + (let [dst (rel dest path)] + (util/dbug* "Filesystem: linking %s...\n" (string/join "/" path)) + (try (Files/deleteIfExists dst) + (Files/createLink (doto dst mkparents!) src) + (when (and mode (not windows?)) (Files/setPosixFilePermissions dst mode)) + (catch java.nio.file.NoSuchFileException ex + (util/dbug* "Filesystem: %s\n" (str ex)))))) + +(defn delete! + [dest path] + (util/dbug* "Filesystem: deleting %s...\n" (string/join "/" path)) + (try (Files/delete (rel dest path)) + (catch java.nio.file.NoSuchFileException ex + (util/dbug* "Filesystem: %s\n" (str ex))))) + +(defn write! + [dest writer-fn path] + (let [dst (rel dest path)] + (mkparents! dst) + (with-open [os (Files/newOutputStream dst open-opts)] + (util/dbug* "Filesystem: writing %s...\n" (string/join "/" path)) + (writer-fn os)))) + +(defn patch! + [dest before after & {:keys [link mode]}] + (with-let [_ (fsp/patch-result before after)] + (doseq [[op path & [arg1 arg2]] (fsp/patch before after link)] + (case op + :delete (delete! dest path) + :write (copy! dest path arg1 arg2 :mode mode) + :link (link! dest path arg1 :mode mode) + :touch (touch! dest path arg1))))) diff --git a/boot/fileset/src/boot/tmpdir.clj b/boot/fileset/src/boot/tmpdir.clj new file mode 100644 index 00000000..0f841364 --- /dev/null +++ b/boot/fileset/src/boot/tmpdir.clj @@ -0,0 +1,388 @@ +(ns boot.tmpdir + (:refer-clojure :exclude [time hash]) + (:require + [clojure.java.io :as io] + [clojure.set :as set] + [clojure.data :as data] + [boot.filesystem :as fs] + [boot.filesystem.patch :as fsp] + [boot.pod :as pod] + [boot.file :as file] + [boot.util :as util :refer [with-let]]) + (:import + [java.io File] + [java.util Properties] + [java.nio.file Path Files SimpleFileVisitor])) + +(set! *warn-on-reflection* true) + +(def CACHE_VERSION "1.0.0") +(def state (atom {:prev {} :cache {}})) + +;; records and protocols ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defprotocol ITmpFile + (id [this]) + (dir [this]) + (bdir [this]) + (path [this]) + (hash [this]) + (time [this]) + (file [this])) + +(defprotocol ITmpFileSet + (ls [this]) + (commit! [this]) + (rm [this paths]) + (add [this dest-dir src-dir opts]) + (add-tmp [this dest-dir tmpfiles]) + (add-cached [this dest-dir cache-key cache-fn opts]) + (mv [this from-path to-path]) + (cp [this src-file dest-tmpfile])) + +(defrecord TmpFile [dir bdir path id hash time] + ITmpFile + (id [this] id) + (dir [this] dir) + (bdir [this] bdir) + (path [this] path) + (hash [this] hash) + (time [this] time) + (file [this] (io/file dir path))) + +(defrecord TmpDir [dir user input output] + ITmpFile + (id [this] nil) + (dir [this] dir) + (bdir [this] nil) + (path [this] "") + (hash [this] "") + (time [this] 0) + (file [this] dir)) + +;; helper functions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn- file-stat + [^File f] + (let [h (file/md5 f) + t (file/last-modified f)] + {:id (str h "." t) :hash h :time t})) + +(defn- scratch-dir! + [^File scratch] + (file/tmpdir scratch "boot-scratch")) + +(def ^:dynamic *hard-link* nil) + +(defn- add-blob! + [^File blob ^Path src ^String id link] + (let [blob (.toPath blob) + out (.resolve blob id)] + (when-not (Files/exists out fs/link-opts) + (if link + (Files/createLink out src) + (let [name (str (.getName out (dec (.getNameCount out)))) + tmp (Files/createTempFile blob name nil fs/tmp-attrs)] + (Files/copy src tmp fs/copy-opts) + (Files/move tmp out fs/copy-opts)))))) + +(defn- mkvisitor + [^Path root ^File blob tree link] + (let [m {:dir (.toFile root) :bdir blob}] + (proxy [SimpleFileVisitor] [] + (visitFile [^Path path attr] + (with-let [_ fs/continue] + (let [p (str (.relativize root path))] + (try (let [h (file/md5 (.toFile path)) + t (.toMillis (Files/getLastModifiedTime path fs/link-opts)) + i (str h "." t)] + (add-blob! blob path i link) + (swap! tree assoc p (map->TmpFile (assoc m :path p :id i :hash h :time t)))) + (catch java.nio.file.NoSuchFileException _ + (util/dbug* "Tmpdir: file not found: %s\n" (.toString p))))))) + (visitFileFailed [^Path path ^java.io.IOException e] + (with-let [_ fs/skip-subtree] + (util/dbug* "Tmpdir: failed to visit: %s\n" (str (.relativize root path)))))))) + +(defn- dir->tree! + [^File dir ^File blob] + (let [root (.toPath dir)] + @(with-let [tree (atom {})] + (file/walk-file-tree root (mkvisitor root blob tree *hard-link*))))) + +(defn- ^File cache-dir + [cache-key] + (-> (boot.App/bootdir) + (io/file "cache" "cache" "fileset") + (io/file CACHE_VERSION cache-key))) + +(defn- ^File manifest-file + [cache-key] + (io/file (cache-dir cache-key) "manifest.properties")) + +(defn- read-manifest + [^File manifile ^File bdir] + (with-open [r (io/input-stream manifile)] + (let [p (doto (Properties.) (.load r))] + (-> #(let [id (.getProperty p %2) + hash (subs id 0 32) + time (Long/parseLong (subs id 33)) + m {:id id :path %2 :hash hash :time time :bdir bdir}] + (->> m map->TmpFile (assoc %1 %2))) + (reduce {} (enumeration-seq (.propertyNames p))))))) + +(defn- write-manifest! + [^File manifile manifest] + (with-open [w (io/output-stream manifile)] + (let [p (Properties.)] + (doseq [[path {:keys [id]}] manifest] + (.setProperty p path id)) + (.store p w nil)))) + +(defn- apply-mergers! + [mergers ^File old-file path ^File new-file ^File merged-file] + (when-let [merger (some (fn [[re v]] (when (re-find re path) v)) mergers)] + (util/dbug* "Merging duplicate entry (%s)\n" path) + (let [out-file (File/createTempFile (file/name merged-file) nil + (file/parent merged-file))] + (with-open [curr-stream (io/input-stream old-file) + new-stream (io/input-stream new-file) + out-stream (io/output-stream out-file)] + (merger curr-stream new-stream out-stream)) + (file/move out-file merged-file)))) + +(defn- set-dir + [tree dir] + (reduce-kv #(assoc %1 %2 (assoc %3 :dir dir)) {} tree)) + +(defn- get-cached! + [cache-key seedfn scratch] + (util/dbug* "Adding cached fileset %s...\n" cache-key) + (or (get-in @state [:cache cache-key]) + (let [cache-dir (cache-dir cache-key) + manifile (manifest-file cache-key) + store! #(with-let [m %] + (swap! state assoc-in [:cache cache-key] m))] + (or (and (file/exists? manifile) + (store! (read-manifest manifile cache-dir))) + (let [tmp-dir (scratch-dir! scratch)] + (util/dbug* "Not found in cache: %s...\n" cache-key) + (.mkdirs cache-dir) + (seedfn tmp-dir) + (binding [*hard-link* true] + (let [m (dir->tree! tmp-dir cache-dir)] + (write-manifest! manifile m) + (store! (read-manifest manifile cache-dir))))))))) + +(defn- merge-trees! + [old new mergers scratch] + (with-let [tmp (scratch-dir! scratch)] + (doseq [[path newtmp] new] + (when-let [oldtmp (get old path)] + (util/dbug* "Merging %s...\n" path) + (let [newf (io/file (bdir newtmp) (id newtmp)) + oldf (io/file (bdir oldtmp) (id oldtmp)) + mergef (doto (io/file tmp path) io/make-parents)] + (apply-mergers! mergers oldf path newf mergef)))))) + +(defn- comp-res + [regexes] + (when-let [res (seq regexes)] + (->> (map #(partial re-find %) res) (apply some-fn)))) + +(defn- filter-tree + [tree include exclude] + (let [ex (comp-res exclude) + in (when-let [in (comp-res include)] (complement in)) + rm? (or (and in ex #(or (in %) (ex %))) in ex)] + (if-not rm? tree (reduce-kv #(if (rm? %2) %1 (assoc %1 %2 %3)) {} tree)))) + +(defn- index + [key tree] + (reduce-kv #(assoc %1 (get %3 key) %3) {} tree)) + +(defn- diff-tree + [tree props] + (let [->map #(select-keys % props)] + (reduce-kv #(assoc %1 %2 (->map %3)) {} tree))) + +(defn- diff* + [{t1 :tree :as before} {t2 :tree :as after} props] + (if-not before + {:added (or after {:tree {}}) + :removed (assoc after :tree {}) + :changed (assoc after :tree {})} + (let [props (or (seq props) [:id]) + d1 (diff-tree t1 props) + d2 (diff-tree t2 props) + [x y z] (map (comp set keys) (data/diff d1 d2)) + changed-keys (set/union (set/intersection x y) (set/intersection (set/union x y) z))] + {:added (->> (set/difference y x changed-keys) (select-keys t2) (assoc after :tree)) + :removed (->> (set/difference x y changed-keys) (select-keys t1) (assoc after :tree)) + :changed (->> changed-keys (select-keys t2) (assoc after :tree))}))) + +(defn- fatal-conflict? + [^File dest] + (if (file/directory? dest) + (let [tree (->> dest file-seq reverse)] + (or (not (every? #(file/directory? ^File %) tree)) + (doseq [^File f tree] (file/delete f)))) + (not (let [d (file/parent dest)] + (or (file/directory? d) (.mkdirs d)))))) + +(defn- add-tree-meta + [tree meta] + (if (empty? meta) + tree + (reduce-kv #(assoc %1 %2 (merge %3 meta)) {} tree))) + +;; fileset implementation ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defrecord TmpFileSet [dirs tree blob scratch] + ITmpFileSet + + (ls [this] + (set (vals tree))) + + (commit! [this] + (let [{:keys [dirs tree blob]} this + prev (get-in @state [:prev dirs]) + {:keys [added removed changed]} (diff* prev this [:id :dir])] + (util/dbug* "Committing fileset...\n") + (doseq [tmpf (set/union (ls removed) (ls changed)) + :let [prev (get-in prev [:tree (path tmpf)]) + exists? (file/exists? ^File (file prev)) + op (if exists? "removing" "no-op")]] + (util/trace* "Commit: %-8s %s %s...\n" op (id prev) (path prev)) + (when exists? (file/delete (file prev)))) + (let [this (loop [this this + [tmpf & tmpfs] + (->> (set/union (ls added) (ls changed)) + (sort-by (comp count path) >))] + (or (and (not tmpf) this) + (let [p (path tmpf) + dst (file tmpf) + src (io/file (bdir tmpf) (id tmpf)) + err? (fatal-conflict? dst) + this (or (and (not err?) this) + (update-in this [:tree] dissoc p))] + (if err? + (util/warn "Merge conflict: not adding %s\n" p) + (do (util/trace* "Commit: adding %s %s...\n" (id tmpf) p) + (file/hard-link src dst))) + (recur this tmpfs))))] + (with-let [_ this] + (swap! state assoc-in [:prev dirs] this) + (util/dbug* "Commit complete.\n"))))) + + (rm [this tmpfiles] + (let [{:keys [dirs tree blob]} this + treefiles (set (vals tree)) + remove? (->> tmpfiles set (set/difference treefiles) complement)] + (assoc this :tree (reduce-kv #(if (remove? %3) %1 (assoc %1 %2 %3)) {} tree)))) + + (add [this dest-dir src-dir opts] + (assert ((set (map file dirs)) dest-dir) + (format "dest-dir not in dir set (%s)" dest-dir)) + (let [{:keys [dirs tree blob scratch]} this + {:keys [mergers include exclude meta]} opts + ->tree #(set-dir (dir->tree! % blob) dest-dir) + new-tree (-> (->tree src-dir) + (filter-tree include exclude) + (add-tree-meta meta)) + mrg-tree (when mergers + (->tree (merge-trees! tree new-tree mergers scratch)))] + (assoc this :tree (merge-with merge tree new-tree mrg-tree)))) + + (add-cached [this dest-dir cache-key cache-fn opts] + (assert ((set (map file dirs)) dest-dir) + (format "dest-dir not in dir set (%s)" dest-dir)) + (let [{:keys [dirs tree blob scratch]} this + {:keys [mergers include exclude meta]} opts + new-tree (let [cached (get-cached! cache-key cache-fn scratch)] + (-> (set-dir cached dest-dir) + (filter-tree include exclude) + (add-tree-meta meta))) + mrg-tree (when mergers + (let [merged (merge-trees! tree new-tree mergers scratch)] + (set-dir (dir->tree! merged blob) dest-dir)))] + (assoc this :tree (merge tree new-tree mrg-tree)))) + + (add-tmp [this dest-dir tmpfiles] + (assert ((set (map file dirs)) dest-dir) + (format "dest-dir not in dir set (%s)" dest-dir)) + (reduce #(assoc-in %1 [:tree (path %2)] (assoc %2 :dir dest-dir)) this tmpfiles)) + + (mv [this from-path to-path] + (if (= from-path to-path) + this + (if-let [from (get-in this [:tree from-path])] + (update-in this [:tree] #(-> % (assoc to-path (assoc from :path to-path)) + (dissoc from-path))) + (throw (Exception. (format "not in fileset (%s)" from-path)))))) + + (cp [this src-file dest-tmpfile] + (let [hash (file/md5 src-file) + p' (path dest-tmpfile) + d' (dir dest-tmpfile)] + (assert ((set (map file dirs)) d') + (format "dest-dir not in dir set (%s)" d')) + (add-blob! blob (file/path src-file) hash *hard-link*) + (assoc this :tree (merge tree {p' (assoc dest-tmpfile :id hash)}))))) + +;; additional api functions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn tmpfile? + [x] + (instance? TmpFile x)) + +(defn tmpfileset? + [x] + (instance? TmpFileSet x)) + +(defn diff + [before after & props] + (let [{:keys [added changed]} + (diff* before after props)] + (update-in added [:tree] merge (:tree changed)))) + +(defn fileset-patch + [before after link] + (let [{:keys [added removed changed]} + (diff* before after [:hash :time]) + ->p #(fs/path->segs (fs/->path %)) + link (#{:tmp :all} link) ; only nil, :tmp, or :all are valid + link? #(and link (or (= :all link) (= (:blob after) (:bdir %))))] + (-> (for [x (->> removed :tree vals)] [:delete (->p (path x))]) + (into (for [x (->> added :tree vals)] + (let [p (file/path ^File (file x))] + [(if (link? x) :link :write) (->p (path x)) p (time x)]))) + (into (for [x (->> changed :tree vals)] + (let [p (file/path ^File (file x)) + x' (get-in before [:tree (path x)])] + (cond (= (hash x) (hash x')) [:touch (->p (path x)) (time x)] + (link? x) [:link (->p (path x)) p (time x)] + :else [:write (->p (path x)) p (time x)]))))))) + +(defmethod fsp/patch TmpFileSet + [before after link] + (fileset-patch before after link)) + +(defn removed + [before after & props] + (:removed (diff* before after props))) + +(defn added + [before after & props] + (:added (diff* before after props))) + +(defn changed + [before after & props] + (:changed (diff* before after props))) + +(defn restrict-dirs + [fileset allowed-dirs] + (let [dirs (set allowed-dirs) + reducer (fn [xs k {:keys [dir] :as v}] + (if-not (dirs dir) xs (assoc xs k v)))] + (update-in fileset [:tree] (partial reduce-kv reducer {})))) diff --git a/boot/fileset/version.properties b/boot/fileset/version.properties new file mode 100644 index 00000000..6587464b --- /dev/null +++ b/boot/fileset/version.properties @@ -0,0 +1 @@ +VERSION=3.0.0-SNAPSHOT From b8d9301df5344dbbd88b1b774c91ab1a621b0536 Mon Sep 17 00:00:00 2001 From: Matthew Ratzke Date: Sat, 6 Jul 2019 21:34:01 -0600 Subject: [PATCH 21/21] add initial tools.deps entrypoints --- boot/pod/src/boot/pod/deps.clj | 81 ++++++++++++++++++++++++++++++++ boot/pod/src/boot/tools/deps.clj | 70 +++++++++++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 boot/pod/src/boot/pod/deps.clj create mode 100644 boot/pod/src/boot/tools/deps.clj diff --git a/boot/pod/src/boot/pod/deps.clj b/boot/pod/src/boot/pod/deps.clj new file mode 100644 index 00000000..cfdc9bd2 --- /dev/null +++ b/boot/pod/src/boot/pod/deps.clj @@ -0,0 +1,81 @@ +(ns boot.pod.deps + (:require [boot.core :as boot] + [boot.pod :as pod])) + +(defn- make-pod + "Make and return a Boot pod with tools.deps.alpha and a recent Clojure + release (to ensure that tools.deps.alpha will run). The pod will also + include Boot and boot-tools-deps (since they're both running), as well as + any other 'runtime' dependencies (which is why it's best to avoid putting + additional dependencies in build.boot when using this tool!)." + [] + (let [pod-env + (-> (boot/get-env) + ;; Pod class path needs to be bootstrapped independently of + ;; the core Pod (build.boot context) so that, for example, + ;; an older version of Clojure than 1.9 can be used in the + ;; core Pod. + (dissoc :boot-class-path :fake-class-path) + ;; Clojure version in the core Pod (build.boot context) + ;; is not guaranteed to be recent enough as Boot supports + ;; 1.6.0 onwards. If it isn't recent enough, we replace it. + ;; We also force tools.deps.alpha to a fixed version. + (update :dependencies ensure-recent-clojure-tools-deps))] + (pod/make-pod pod-env))) + +(defn build-environment-map + "Run tools.deps to produce: + * :resource-paths -- source code directories from :paths in deps.edn files + * :source-paths -- additional directories from :extra-paths and classpath + * :repositories -- any entries from :mvn/repos + * :dependencies -- vector of Maven coordinates + * :classpath -- JAR files to add to the classpath + * :main-opts -- any main-opts pulled from tools.deps.alpha" + [{:keys [config-data ; no config-paths + classpath-aliases main-aliases resolve-aliases + verbose ; no repeatable + system-deps deps-files total] + :as options}] + (let [deps (reader/read-deps + (into [] (comp (map io/file) + (filter #(.exists %))) + deps-files)) + deps (if total + (if config-data + (reader/merge-deps [deps config-data]) + deps) + (reader/merge-deps + (cond-> [system-deps deps] + config-data (conj config-data)))) + paths (set (or (seq (:paths deps)) [])) + resolve-args (cond-> + (deps/combine-aliases deps resolve-aliases) + ;; handle both legacy boolean and new counter + (and verbose + (or (boolean? verbose) + (< 1 verbose))) + (assoc :verbose true)) + libs (deps/resolve-deps deps resolve-args) + cp-args (deps/combine-aliases deps classpath-aliases) + cp (deps/make-classpath libs (:paths deps) cp-args) + main-opts (:main-opts (deps/combine-aliases deps main-aliases)) + cp-separator (re-pattern java.io.File/pathSeparator) + [jars dirs] (reduce (fn [[jars dirs] item] + (let [f (java.io.File. item)] + (if (and (.exists f) + (not (paths item))) + (cond (.isFile f) + [(conj jars item) dirs] + (.isDirectory f) + [jars (conj dirs item)] + :else + [jars dirs]) + [jars dirs]))) + [[] []] + (str/split cp cp-separator))] + {:resource-paths paths + :source-paths (set dirs) + :repositories (:mvn/repos deps) + :dependencies libs + :classpath jars + :main-opts main-opts})) diff --git a/boot/pod/src/boot/tools/deps.clj b/boot/pod/src/boot/tools/deps.clj new file mode 100644 index 00000000..aeeac8e9 --- /dev/null +++ b/boot/pod/src/boot/tools/deps.clj @@ -0,0 +1,70 @@ +(ns boot.tools.deps + "Dependency management using tools.deps." + {:boot/export-tasks true}) + +(deftask deps + "Use tools.deps to read and resolve the specified deps.edn files. + + The dependencies read in are added to your Boot classpath. There are two + options to update Boot's :dependencies if additional tasks in your pipeline + rely on that: -B will completely overwrite the Boot :dependencies with the + ones produced by tools.deps and should be used when you are creating uber + jars; -Q will perform a quick merge of the dependencies from tools.deps into + the Boot environment and may be needed for certain testing tools. + + As much as possible, the recommended approach is to avoid using the Boot + :dependencies vector when using boot-tools-deps so that deps.edn represents + the total dependencies for your project. + + Most of the arguments are intended to match the clj script usage + (as passed to clojure.tools.deps.alpha.script.make-classpath/-main). + + In particular, the -c / --config-paths option is assumed to be the COMPLETE + list of EDN files to read (and therefore overrides the default set of + system deps, user deps, and local deps). + + The -r option is intended to be equivalent to the -Srepro option in + tools.deps, which will exclude both the system deps and the user deps. + + The -D option is intended to be the equivalent to the -Sdeps option, which + allows you to specify an additional deps.edn-like map of dependencies which + are added to the set of deps.edn-derived dependencies (even when -r is + given). + + The -A, -C, -M, and -R options mirror the clj script usage for aliases. + + The -x option will run clojure.main with any main-opts found by deps.edn. + + The -v option makes boot-tools-deps verbose, explaining which files it looked + for, the dependencies it got back from tools.dep, and the changes it made to + Boot's classpath, :resource-paths, and :source-path. If you specify it twice + (-vv) then tools.deps will also be verbose about its work." + + [;; options that mirror tools.deps itself: + c config-paths PATH [str] "the list of deps.edn files to read." + r repeatable bool "Use only the specified deps.edn file for a repeatable build." + D config-data EDN edn "is treated as a final deps.edn file." + A aliases KW [kw] "the list of aliases (for -C, -M, and -R)." + C classpath-aliases KW [kw] "the list of classpath aliases to use." + M main-aliases KW [kw] "the list of main-opt aliases to use." + R resolve-aliases KW [kw] "the list of resolve aliases to use." + ;; options specific to boot-tools-deps + B overwrite-boot-deps bool "Overwrite Boot's :dependencies." + Q quick-merge bool "Merge into Boot's :dependencies." + v verbose int "the verbosity level." + x execute bool "Execute clojure.main with any main-opts found."] + (let [{:keys [main-opts]} + (load-deps {:config-paths config-paths + :config-data config-data + :classpath-aliases (into (vec aliases) classpath-aliases) + :main-aliases (into (vec aliases) main-aliases) + :resolve-aliases (into (vec aliases) resolve-aliases) + :overwrite-boot-deps overwrite-boot-deps + :quick-merge quick-merge + :repeatable repeatable + :verbose verbose})] + (boot/with-pass-thru fs + (when execute + (when verbose + (println "Executing clojure.main" (str/join " " main-opts))) + (apply clojure.main/main main-opts)))))