testing cluster mode

Author: alekcz

README.md

@@ -252,7 +252,7 @@ You can see the spike in CPU and bandwidth on the droplet, but the CPU never sat
 These are by no means comprehensive benchmarks but give you a sense of what a PCP server can withstand for simple use case.
 Going above 400 req/s generally results in bad things happening. You can test your own site using [k6](https://k6.io/) with the instructions in [loadtest.js](./loadtest.js)
 
-# Roadmap
+# Roadmap & releases
 
 ## Release 0.0.1
 - [x] Run arbitrary scripts
@@ -268,6 +268,10 @@ Going above 400 req/s generally results in bad things happening. You can test yo
 - [x] Document PCP
 - [x] Load test a production deployment
 
+## Release 0.0.2
+- [ ] Perfomance improvements
+- [ ] Cluster mode
+
 ## Release 0.1.0
 - [ ] Store passphrases with [konserve](https://github.com/replikativ/konserve)
 - [ ] Add sponsorship button
@@ -285,6 +289,7 @@ For the guidance and examples, special thanks to
 
 - [@BrunoBonacci](https://github.com/BrunoBonacci) 
 - [@borkdude](https://github.com/borkdude) 
+- [@Baeldung](https://twitter.com/Baeldung)
 
 ## License
 

install.sh

@@ -53,6 +53,10 @@ mv -f "$download_dir/pcp-server.jar" "$PWD/pcp-server.jar"
 rm -rf "$PWD/pcp-templates"
 mv -f "$download_dir/pcp-templates" "$PWD"
 
+if [ ! -e "$PWD/pcp.conf" ] ; then
+    echo "PCP_CLUSTER=-c" > "$PWD/pcp.conf"
+fi
+
 case "$(uname -s)" in
     Linux*)     
         mv -f "$download_dir/pcp.service" "/etc/systemd/system/pcp.service"

project.clj

@@ -1,4 +1,4 @@
-(defproject pcp "0.0.2-beta.1"
+(defproject pcp "0.0.2-beta.2"
   :description "PCP: Clojure Processor - A Clojure replacement for PHP"
   :url "https://github.com/alekcz/pcp"
   :license {:name "The MIT License"
@@ -7,6 +7,7 @@
   :dependencies [ ;core
                   [org.clojure/clojure "1.10.3"]
                   [org.clojure/tools.cli "1.0.194"]
+                  [org.clojure/core.async "1.3.618"]
                   [borkdude/sci "0.2.5"]
                   [byte-streams "0.2.4"]
                   [http-kit "2.5.0-RC1"]

resources/PCP_VERSION

@@ -1 +1 @@
-v0.0.2-beta.1
+v0.0.2-beta.2

resources/pcp.service

@@ -2,7 +2,8 @@
 Description= pcp scgi server
 
 [Service]
-ExecStart=/usr/bin/java -jar /usr/local/bin/pcp-server.jar
+EnvironmentFile=/usr/local/bin/pcp.conf
+ExecStart=/usr/bin/java -jar /usr/local/bin/pcp-server.jar $PCP_CLUSTER
 Restart=always
 
 [Install]

src/pcp/core.clj

@@ -89,8 +89,8 @@
     (catch Exception _ nil)))
 
 (defn valid-path? [parent target]
-  (let [parent-path (-> parent io/file (.getParentFile) (.getCanonicalPath))
-        target-path (-> target io/file (.getCanonicalPath))]
+  (let [parent-path (-> parent ^File io/file (.getParentFile) (.getCanonicalPath))
+        target-path (-> target ^File io/file (.getCanonicalPath))]
     (str/starts-with? target-path parent-path)))
 
 (def persist ^:sci/macro
@@ -105,9 +105,9 @@
 (defn run-script [url-path &{:keys [root request]}]
   (let [path (URLDecoder/decode url-path "UTF-8")
         source (read-source path)
-        file (io/file path)
-        root (or root (-> file (.getParentFile) (.getCanonicalPath)))
-        parent (longer root (-> file (.getParentFile) (.getCanonicalPath)))
+        ^File file (io/file path)
+        root (or root (-> ^File file (.getParentFile) (.getCanonicalPath)))
+        parent (longer root (-> ^File file (.getParentFile) (.getCanonicalPath)))
         response (atom nil)
         keygen (fn [path k] (keyword (str (h/uuid [path k]))))]     
     (if (string? source)
@@ -200,7 +200,16 @@
   ([path]       
     (let [scgi-port (Integer/parseInt (or (System/getenv "SCGI_PORT") "9000"))]
       (case path
-        "" (scgi/serve scgi-handler scgi-port)
+        "" 
+          (scgi/serve scgi-handler scgi-port)
+        
+        "-c"  
+          (do 
+            (scgi/serve scgi-handler scgi-port)
+            (scgi/serve scgi-handler 9007)
+            (scgi/serve scgi-handler 9014)
+            (scgi/serve scgi-handler 9021))              
+        
         (run-script path)))))
 
 

src/pcp/resp.clj

@@ -51,6 +51,7 @@
     :.bmp "image/bmp"
     :.bz "application/x-bzip"
     :.bz2 "application/x-bzip2"
+    :.cljs "application/x-scittle"
     :.csh "application/x-csh"
     :.css "text/css"
     :.csv "text/csv"

src/pcp/scgi.clj

@@ -17,29 +17,29 @@
   (let [data (str/split (:header req) #"\u0000")
         keys (map #(-> % (str/replace "_" "-") str/lower-case keyword) (take-nth 2 data))
         values (take-nth 2 (rest data))
-        h (zipmap keys values)]
+        h (transient (zipmap keys values))]
     ;make the ring linter happy.
     (-> h
-      (update :server-port #(Integer/parseInt (if (str/blank? %) "0" %)))
-      (update :content-length #(Integer/parseInt (if (str/blank? %) "0" %)))
-      (update :request-method #(-> % str str/lower-case keyword))
-      (assoc  :headers { "sec-fetch-site" (-> h :http-sec-fetch-site)   
-                        "host" (-> h :http-host)   
-                        "user-agent" (-> h :http-user-agent)     
-                        "cookie" (-> h :http-cookie)   
-                        "sec-fetch-user" (-> h :http-sec-fetch-user)   
-                        "connection" (-> h :hhttp-connection)   
-                        "upgrade-insecure-requests" (-> h :http-sec-fetch-site)   
-                        "accept"  (-> h :http-accept)   
-                        "accept-language"   (-> h :http-accept-language)   
-                        "sec-fetch-dest" (-> h :http-sec-fetch-dest)   
-                        "accept-encoding" (-> h :http-accept-encoding)   
-                        "sec-fetch-mode" (-> h :http-sec-fetch-mode)    
-                        "cache-control" (-> h :http-cache-control)})
-      (assoc :headers {})
-      (assoc :uri (:request-uri h))
-      (assoc :scheme (-> h :request-scheme keyword))
-      (assoc :body (:body req)))))
+      (assoc! :server-port (Integer/parseInt (if (str/blank? (:server-port h)) "0" (:server-port h))))
+      (assoc! :content-length (Integer/parseInt (if (str/blank? (:content-length h)) "0" (:content-length h))))
+      (assoc! :request-method (-> (:request-method h) str str/lower-case keyword))
+      (assoc! :headers {  "sec-fetch-site" (-> h :http-sec-fetch-site)   
+                          "host" (-> h :http-host)   
+                          "user-agent" (-> h :http-user-agent)     
+                          "cookie" (-> h :http-cookie)   
+                          "sec-fetch-user" (-> h :http-sec-fetch-user)   
+                          "connection" (-> h :hhttp-connection)   
+                          "upgrade-insecure-requests" (-> h :http-sec-fetch-site)   
+                          "accept"  (-> h :http-accept)   
+                          "accept-language"   (-> h :http-accept-language)   
+                          "sec-fetch-dest" (-> h :http-sec-fetch-dest)   
+                          "accept-encoding" (-> h :http-accept-encoding)   
+                          "sec-fetch-mode" (-> h :http-sec-fetch-mode)    
+                          "cache-control" (-> h :http-cache-control)})
+      (assoc! :uri (:request-uri h))
+      (assoc! :scheme (-> h :request-scheme keyword))
+      (assoc! :body (:body req))
+      (persistent!))))
 
 (defn on-accept [^SelectionKey key]
   (let [^ServerSocketChannel channel     (.channel key)

src/pcp/utility.clj

@@ -10,7 +10,7 @@
     [taoensso.nippy :as nippy]
     [environ.core :refer [env]])
   (:import  [java.net Socket]
-            [java.io File ByteArrayOutputStream InputStream]
+            [java.io File ByteArrayOutputStream InputStream BufferedWriter]
             [org.apache.commons.io IOUtils]
             [org.apache.commons.codec.digest DigestUtils]) 
   (:gen-class))
@@ -19,7 +19,7 @@
 
 (def root (atom nil))
 (def scgi (atom "9000"))
-(def version "v0.0.2-beta.1")
+(def version "v0.0.2-beta.2")
 
 (defn keydb []
   (or (env :pcp-keydb) "/usr/local/etc/pcp-db"))
@@ -97,8 +97,8 @@ Options:
       (resp/status status)
       (resp/content-type mime-type))) 
 
-(defn file-response [path ^File file]
-  (let [code (if (.exists file) 200 404)
+(defn file-response [path file]
+  (let [code (if (.exists ^File file) 200 404)
         mime (resp/get-mime-type (re-find #"\.[0-9A-Za-z]{1,7}$" path))]
     (-> (resp/response file)    
         (resp/status code)
@@ -115,14 +115,14 @@ Options:
     {:status status :body body :headers headers}))
 
 (defn file-exists? [path]
-  (-> path io/file .exists))
+  (-> path ^File io/file .exists))
 
 (defn serve-file [path]
   (file-response path (io/file path)))
 
 (defn local-handler [opts]
   (fn [request]
-    (let [root (.getCanonicalPath (io/file (:root opts)))
+    (let [root (.getCanonicalPath ^File (io/file (:root opts)))
           path (str root (:uri request))
           slashpath (str path "index.clj")
           exists (or (file-exists? path) (file-exists? slashpath))
@@ -141,7 +141,7 @@ Options:
 
 (defn run-file [path port]
   (let [path (str/replace (str "/" path) "//" "/")
-        root (.getCanonicalPath (io/file "./"))
+        root (.getCanonicalPath ^File (io/file "./"))
         scgi-port (Integer/parseInt (or (System/getenv "SCGI_PORT") (str port) "9000"))
         request {:document-root root :document-uri path :request-method :get}]
     (-> request http-to-scgi (forward scgi-port) create-resp :body)))
@@ -235,7 +235,7 @@ Options:
         path (str (keydb) "/" project ".db")]
   (io/make-parents (keydb))
   (println "adding passphrase for project:" project)
-  (with-open [w (io/writer path)]
+  (with-open [^BufferedWriter w (io/writer path)]
     (.write w ^String passphrase))
   (println "done.")))  
 
@@ -256,7 +256,7 @@ Options:
     (spit 
       (str path "/public/api/info.clj") 
       (slurp (str (template-path) "/api/info.clj")))
-    (println (str "Created pcp project `" project-name "` in directory") (.getAbsolutePath (io/file path)))))
+    (println (str "Created pcp project `" project-name "` in directory") (.getAbsolutePath ^File (io/file path)))))
 
 (defn -main 
   ([]

test/pcp/core_test.clj

@@ -4,8 +4,7 @@
             [pcp.resp :as resp]
             [pcp.core :as core]
             [clojure.string :as str])
-  (:import  [java.io File]
-            [java.net Socket InetAddress ConnectException]))
+  (:import  [java.io File]))
 
 (deftest read-source-test
   (testing "Test reading source"
@@ -100,5 +99,4 @@
           scgi-request {:document-root root :document-uri uri }
           _ (core/scgi-handler scgi-request)
           ans  (try (core/run-script (str root uri)) (catch Exception _ "error"))]
-    (is (str/includes? ans "error")))))
-    
+    (is (str/includes? ans "error")))))