Gzip descriptor files

Author: kennyjwilli

env/update_descriptors.clj

@@ -1,32 +1,36 @@
 (ns update-descriptors
   (:require
-    [clojure.string :as str]
-    [clojure.java.io :as io]
     [clojure.data.json :as json]
-    [java-http-clj.core :as http]
-    [compute.gcp.impl.response :as response]
-    [compute.gcp.impl.discovery-doc :as discovery-doc]
+    [clojure.java.io :as io]
+    [clojure.spec.alpha :as s]
+    [clojure.string :as str]
     [compute.gcp.descriptor :as descriptor]
-    [clojure.spec.alpha :as s]))
+    [compute.gcp.impl.discovery-doc :as discovery-doc]
+    [compute.gcp.impl.response :as response]
+    [java-http-clj.core :as http])
+  (:import (java.util.zip GZIPOutputStream)))
 
 (defn get-json
   [url]
   (let [resp (http/get url {:timeout 5000} {:as :input-stream})]
     (if (= 200 (:status resp))
       (json/read (io/reader (:body resp)))
       (throw (ex-info (str "Failed getting JSON " (:status resp))
-                      {:cognitect.anomalies/category (response/http-status->category (:status resp))
-                       :resp                         resp})))))
+               {:cognitect.anomalies/category (response/http-status->category (:status resp))
+                :resp                         resp})))))
 
 (defn list-discovery-docs
   []
   (-> (get-json "https://www.googleapis.com/discovery/v1/apis")
-      (get "items")))
+    (get "items")))
 
 (defn spit-with-dirs
-  [path content]
+  [{:keys [path content output-stream-fn]}]
   (.mkdirs (.getParentFile (io/file path)))
-  (spit path content))
+  (with-open [os (if output-stream-fn
+                   (output-stream-fn path)
+                   (io/output-stream (io/file path)))]
+    (io/copy content os)))
 
 (defn download-all-discovery-docs
   [discovery-docs-list]
@@ -39,7 +43,7 @@
           (merge {:cognitect.anomalies/category :cognitect.anomalies/fault
                   :id                           id
                   :ex                           ex}
-                 (ex-data ex)))))
+            (ex-data ex)))))
     discovery-docs-list))
 
 (defn convert-one-download
@@ -57,37 +61,41 @@
   ([downloads base-dir]
    (let [{failed-download-anoms true
           downloads             false} (group-by (comp boolean :cognitect.anomalies/category)
-                                                 downloads)
+                                         downloads)
          converted (map convert-one-download downloads)
          {convert-anoms true
           successes     false} (group-by (comp boolean :cognitect.anomalies/category)
-                                         converted)
+                                 converted)
          anoms (concat failed-download-anoms convert-anoms)]
 
      ;; Print anomalies
      (doseq [[anom-category anoms] (group-by :cognitect.anomalies/category anoms)]
        (println (format "[Warning] %s Swagger files failed with anomaly %s: %s"
-                        (count anoms)
-                        (name anom-category)
-                        (str/join ", " (map :id anoms)))))
+                  (count anoms)
+                  (name anom-category)
+                  (str/join ", " (map :id anoms)))))
 
      (let [successes (map (fn [{::keys [descriptor] :as m}]
                             (assoc m
                               ::lib-dir (io/file
                                           (::descriptor/name descriptor)
                                           (::descriptor/api-version descriptor))))
-                          successes)
+                       successes)
            all-deps-edn-content {:paths (mapv (comp (memfn getPath) ::lib-dir) successes)}]
        (spit-with-dirs
-         (io/file base-dir "deps.edn")
-         (pr-str all-deps-edn-content))
+         {:path    (io/file base-dir "deps.edn")
+          :content (pr-str all-deps-edn-content)})
 
        (doseq [{::keys [descriptor lib-dir]} successes
-               :let [{:keys [::descriptor/name ::descriptor/api-version]} descriptor
+               :let [{::descriptor/keys [name api-version]} descriptor
                      deps-edn-path (io/file lib-dir "deps.edn")
-                     descriptor-path (io/file lib-dir (descriptor/api-descriptor-resource-path name api-version))]]
-         (spit-with-dirs (io/file base-dir descriptor-path) descriptor)
-         (spit-with-dirs (io/file base-dir deps-edn-path) (pr-str {:paths ["."]})))))))
+                     descriptor-path (io/file lib-dir (descriptor/api-descriptor-resource-path name api-version))
+                     descriptor-output-file (io/file base-dir descriptor-path)]]
+         (spit-with-dirs {:path             descriptor-output-file
+                          :content          (pr-str descriptor)
+                          :output-stream-fn #(GZIPOutputStream. (io/output-stream %))})
+         (spit-with-dirs {:path    (io/file base-dir deps-edn-path)
+                          :content (pr-str {:paths ["."]})}))))))
 
 (comment
   (def discovery-docs-list (list-discovery-docs))
@@ -99,12 +107,12 @@
   (def discovery-docs (map ::discovery-doc downloads))
 
   (into []
-        (comp
-          (filter (fn [doc]
-                    (not-empty (get doc "servicePath"))))
-          (map (fn [doc]
-                 (select-keys doc ["servicePath" "name"]))))
-        discovery-docs)
+    (comp
+      (filter (fn [doc]
+                (not-empty (get doc "servicePath"))))
+      (map (fn [doc]
+             (select-keys doc ["servicePath" "name"]))))
+    discovery-docs)
 
   ;; all paths
   (for [doc discovery-docs
@@ -137,4 +145,4 @@
         downloads (download-all-discovery-docs discovery-docs-list)]
     (update! downloads descriptors-dir)
     (println "Success!")
-    (shutdown-agents)))
+    (shutdown-agents)))

src/compute/gcp/descriptor.clj

@@ -1,9 +1,9 @@
 (ns compute.gcp.descriptor
   (:require
-    [clojure.spec.alpha :as s]
-    [clojure.java.io :as io]
     [clojure.edn :as edn]
-    [clojure.walk :as walk]))
+    [clojure.java.io :as io]
+    [clojure.spec.alpha :as s])
+  (:import (java.util.zip GZIPInputStream)))
 
 ;; TODO
 (s/def ::json-schema map?)
@@ -66,7 +66,7 @@
 
 (defn api-descriptor-resource-path
   [api api-version]
-  (format "%s/%s/%s/api-descriptor.edn"
+  (format "%s/%s/%s/api-descriptor.edn.gz"
           api-descriptor-resource-path-base-directory
           (name api)
           api-version))
@@ -100,7 +100,7 @@
 (defn read-descriptor
   [api api-version]
   (if-let [descriptor-resource (io/resource (api-descriptor-resource-path api api-version))]
-    (with-open [rdr (io/reader descriptor-resource)
+    (with-open [rdr (io/reader (GZIPInputStream. (io/input-stream descriptor-resource)))
                 push-rdr (java.io.PushbackReader. rdr)]
       (edn/read push-rdr))
     (throw (ex-info (format "Cannot find API descriptor file for %s %s"

test/compute/gcp/impl/descriptor_test.clj

@@ -5,7 +5,7 @@
     [compute.gcp.descriptor :as descriptor]))
 
 (deftest api-descriptor-resource-path-test
-  (is (= "computesoftware/api-descriptors/compute/v1/api-descriptor.edn"
+  (is (= "computesoftware/api-descriptors/compute/v1/api-descriptor.edn.gz"
          (descriptor/api-descriptor-resource-path :compute "v1"))))