Merge pull request ComputeSoftware#7 from ComputeSoftware/gzip

Gzip descriptors & build.clj

Author: kennyjwilli

.gitignore

@@ -4,4 +4,6 @@
 *.iml
 gcp-api-descriptors
 creds.json
-lib.jar
+lib.jar
+pom.xml
+target

bin/deploy-jar.sh

@@ -2,9 +2,5 @@
 
 set -euo pipefail
 
-echo "building jar..."
-clojure -Spom
-clojure -A:jar
-
-echo "deploying..."
-mvn deploy:deploy-file -Dfile=lib.jar -DrepositoryId=clojars -Durl=https://clojars.org/repo -DpomFile=pom.xml
+clojure -T:build jar
+clojure -T:build deploy

build.clj

@@ -0,0 +1,18 @@
+(ns build
+  (:require
+    [clojure.tools.build.api :as b]
+    [org.corfield.build :as bb]))
+
+(def lib 'gcp-api/gcp-api)
+(def version (format "1.0.%s" (b/git-count-revs nil)))
+
+(defn jar "Build lib jar." [opts]
+  (-> (assoc opts :lib lib :version version)
+    (bb/clean)
+    (bb/jar))
+  opts)
+
+(defn deploy "Deploy the JAR to Clojars." [opts]
+  (-> opts
+    (assoc :lib lib :version version)
+    (bb/deploy)))

deps.edn

@@ -1,20 +1,17 @@
 {:paths   ["src"]
- :deps    {org.clojure/core.async                          {:mvn/version "1.3.610"}
+ :deps    {org.clojure/core.async                          {:mvn/version "1.5.648"}
            com.cognitect/anomalies                         {:mvn/version "0.1.12"}
-           org.clojure/data.json                           {:mvn/version "1.0.0"}
-           com.google.auth/google-auth-library-oauth2-http {:mvn/version "0.21.1"}
-           java-http-clj/java-http-clj                     {:mvn/version "0.4.1"}}
+           org.clojure/data.json                           {:mvn/version "2.4.0"}
+           com.google.auth/google-auth-library-oauth2-http {:mvn/version "1.6.0"}
+           java-http-clj/java-http-clj                     {:mvn/version "0.4.3"}}
  :aliases {:test        {:extra-paths ["test"]
                          :extra-deps  {provisdom/test         {:git/url "https://github.com/Provisdom/test.git"
-                                                               :sha     "f6a10aa33d7cafacd78ea948b99f24b9a6b5ed30"}
-                                       org.clojure/test.check {:mvn/version "1.1.0"}}}
-           :test-runner {:extra-deps {lambdaisland/kaocha {:mvn/version "1.0.672"}}
+                                                               :sha     "c12852d4dbe90ed5a6a8675618cfbf49eed60970"}
+                                       org.clojure/test.check {:mvn/version "1.1.1"}}}
+           :build       {:deps       {io.github.seancorfield/build-clj {:git/tag "v0.8.0" :git/sha "9bd8b8a"}}
+                         :ns-default build}
+           :test-runner {:extra-deps {lambdaisland/kaocha {:mvn/version "1.64.1010"}}
                          :main-opts  ["-m" "kaocha.runner"]}
            :dev         {:extra-paths ["resources" "env" "gcp-api-descriptors"]
-                         :extra-deps  {com.climate/claypoole {:mvn/version "1.1.4"}
-                                       gcp-api/compute       {:git/url   "https://github.com/ComputeSoftware/gcp-api-descriptors.git"
-                                                              :sha       "7b00d7c1ff31b03e9cd7df4189ae8dd8e533eec4"
-                                                              :deps/root "compute/v1"}}}
-           :descriptors {:extra-deps {gcp-api/descriptors {:local/root "gcp-api-descriptors"}}}
-           :jar         {:extra-deps {seancorfield/depstar {:mvn/version "1.1.104"}}
-                         :main-opts  ["-m" "hf.depstar.jar" "lib.jar"]}}}
+                         :extra-deps  {com.climate/claypoole {:mvn/version "1.1.4"}}}
+           :descriptors {:extra-deps {gcp-api/descriptors {:local/root "gcp-api-descriptors"}}}}}

env/update_descriptors.clj

@@ -1,32 +1,36 @@
 (ns update-descriptors
   (:require
-    [clojure.string :as str]
-    [clojure.java.io :as io]
     [clojure.data.json :as json]
-    [java-http-clj.core :as http]
-    [compute.gcp.impl.response :as response]
-    [compute.gcp.impl.discovery-doc :as discovery-doc]
+    [clojure.java.io :as io]
+    [clojure.spec.alpha :as s]
+    [clojure.string :as str]
     [compute.gcp.descriptor :as descriptor]
-    [clojure.spec.alpha :as s]))
+    [compute.gcp.impl.discovery-doc :as discovery-doc]
+    [compute.gcp.impl.response :as response]
+    [java-http-clj.core :as http])
+  (:import (java.util.zip GZIPOutputStream)))
 
 (defn get-json
   [url]
   (let [resp (http/get url {:timeout 5000} {:as :input-stream})]
     (if (= 200 (:status resp))
       (json/read (io/reader (:body resp)))
       (throw (ex-info (str "Failed getting JSON " (:status resp))
-                      {:cognitect.anomalies/category (response/http-status->category (:status resp))
-                       :resp                         resp})))))
+               {:cognitect.anomalies/category (response/http-status->category (:status resp))
+                :resp                         resp})))))
 
 (defn list-discovery-docs
   []
   (-> (get-json "https://www.googleapis.com/discovery/v1/apis")
-      (get "items")))
+    (get "items")))
 
 (defn spit-with-dirs
-  [path content]
+  [{:keys [path content output-stream-fn]}]
   (.mkdirs (.getParentFile (io/file path)))
-  (spit path content))
+  (with-open [os (if output-stream-fn
+                   (output-stream-fn path)
+                   (io/output-stream (io/file path)))]
+    (io/copy content os)))
 
 (defn download-all-discovery-docs
   [discovery-docs-list]
@@ -39,7 +43,7 @@
           (merge {:cognitect.anomalies/category :cognitect.anomalies/fault
                   :id                           id
                   :ex                           ex}
-                 (ex-data ex)))))
+            (ex-data ex)))))
     discovery-docs-list))
 
 (defn convert-one-download
@@ -57,37 +61,41 @@
   ([downloads base-dir]
    (let [{failed-download-anoms true
           downloads             false} (group-by (comp boolean :cognitect.anomalies/category)
-                                                 downloads)
+                                         downloads)
          converted (map convert-one-download downloads)
          {convert-anoms true
           successes     false} (group-by (comp boolean :cognitect.anomalies/category)
-                                         converted)
+                                 converted)
          anoms (concat failed-download-anoms convert-anoms)]
 
      ;; Print anomalies
      (doseq [[anom-category anoms] (group-by :cognitect.anomalies/category anoms)]
        (println (format "[Warning] %s Swagger files failed with anomaly %s: %s"
-                        (count anoms)
-                        (name anom-category)
-                        (str/join ", " (map :id anoms)))))
+                  (count anoms)
+                  (name anom-category)
+                  (str/join ", " (map :id anoms)))))
 
      (let [successes (map (fn [{::keys [descriptor] :as m}]
                             (assoc m
                               ::lib-dir (io/file
                                           (::descriptor/name descriptor)
                                           (::descriptor/api-version descriptor))))
-                          successes)
+                       successes)
            all-deps-edn-content {:paths (mapv (comp (memfn getPath) ::lib-dir) successes)}]
        (spit-with-dirs
-         (io/file base-dir "deps.edn")
-         (pr-str all-deps-edn-content))
+         {:path    (io/file base-dir "deps.edn")
+          :content (pr-str all-deps-edn-content)})
 
        (doseq [{::keys [descriptor lib-dir]} successes
-               :let [{:keys [::descriptor/name ::descriptor/api-version]} descriptor
+               :let [{::descriptor/keys [name api-version]} descriptor
                      deps-edn-path (io/file lib-dir "deps.edn")
-                     descriptor-path (io/file lib-dir (descriptor/api-descriptor-resource-path name api-version))]]
-         (spit-with-dirs (io/file base-dir descriptor-path) descriptor)
-         (spit-with-dirs (io/file base-dir deps-edn-path) (pr-str {:paths ["."]})))))))
+                     descriptor-path (io/file lib-dir (descriptor/api-descriptor-resource-path name api-version))
+                     descriptor-output-file (io/file base-dir descriptor-path)]]
+         (spit-with-dirs {:path             descriptor-output-file
+                          :content          (pr-str descriptor)
+                          :output-stream-fn #(GZIPOutputStream. (io/output-stream %))})
+         (spit-with-dirs {:path    (io/file base-dir deps-edn-path)
+                          :content (pr-str {:paths ["."]})}))))))
 
 (comment
   (def discovery-docs-list (list-discovery-docs))
@@ -99,12 +107,12 @@
   (def discovery-docs (map ::discovery-doc downloads))
 
   (into []
-        (comp
-          (filter (fn [doc]
-                    (not-empty (get doc "servicePath"))))
-          (map (fn [doc]
-                 (select-keys doc ["servicePath" "name"]))))
-        discovery-docs)
+    (comp
+      (filter (fn [doc]
+                (not-empty (get doc "servicePath"))))
+      (map (fn [doc]
+             (select-keys doc ["servicePath" "name"]))))
+    discovery-docs)
 
   ;; all paths
   (for [doc discovery-docs
@@ -137,4 +145,4 @@
         downloads (download-all-discovery-docs discovery-docs-list)]
     (update! downloads descriptors-dir)
     (println "Success!")
-    (shutdown-agents)))
+    (shutdown-agents)))

pom.xml

@@ -1,9 +1,9 @@
 (ns compute.gcp.descriptor
   (:require
-    [clojure.spec.alpha :as s]
-    [clojure.java.io :as io]
     [clojure.edn :as edn]
-    [clojure.walk :as walk]))
+    [clojure.java.io :as io]
+    [clojure.spec.alpha :as s])
+  (:import (java.util.zip GZIPInputStream)))
 
 ;; TODO
 (s/def ::json-schema map?)
@@ -66,7 +66,7 @@
 
 (defn api-descriptor-resource-path
   [api api-version]
-  (format "%s/%s/%s/api-descriptor.edn"
+  (format "%s/%s/%s/api-descriptor.edn.gz"
           api-descriptor-resource-path-base-directory
           (name api)
           api-version))
@@ -100,7 +100,7 @@
 (defn read-descriptor
   [api api-version]
   (if-let [descriptor-resource (io/resource (api-descriptor-resource-path api api-version))]
-    (with-open [rdr (io/reader descriptor-resource)
+    (with-open [rdr (io/reader (GZIPInputStream. (io/input-stream descriptor-resource)))
                 push-rdr (java.io.PushbackReader. rdr)]
       (edn/read push-rdr))
     (throw (ex-info (format "Cannot find API descriptor file for %s %s"

src/compute/gcp/descriptor.clj

@@ -5,7 +5,7 @@
     [compute.gcp.descriptor :as descriptor]))
 
 (deftest api-descriptor-resource-path-test
-  (is (= "computesoftware/api-descriptors/compute/v1/api-descriptor.edn"
+  (is (= "computesoftware/api-descriptors/compute/v1/api-descriptor.edn.gz"
          (descriptor/api-descriptor-resource-path :compute "v1"))))