automation: introduce simple streaming of transcripts to runner script

Author: darwin

src/automation/dirac/automation/task.cljs

@@ -4,6 +4,7 @@
             [oops.core :refer [oget oset! ocall oapply gset!]]
             [chromex.logging :refer-macros [log warn error info]]
             [dirac.settings :refer-macros [get-signal-server-url
+                                           get-transcript-streamer-server-url
                                            get-chrome-remote-debugging-port
                                            get-chrome-remote-debugging-host
                                            get-pending-replies-wait-timeout
@@ -100,6 +101,10 @@
     (runner/normalized?)
     true))
 
+(defn get-transcript-streamer-server-url-if-needed []
+  (if (helpers/automated-testing?)
+    (get-transcript-streamer-server-url)))
+
 (defn make-failure-matcher []
   (fn [[label _message]]
     (if-not (failed?)
@@ -118,7 +123,7 @@
     (messages/init! "task-runner")
     ; transcript is a fancy name for "log of interesting events"
     (register-global-exception-handler!)
-    (transcript-host/init-transcript! "transcript-box" (normalized?))
+    (transcript-host/init-transcript! "transcript-box" (normalized?) (get-transcript-streamer-server-url-if-needed))
     ; when we are not running under test-runner, we want skip all future actions after a failure
     ; this helps inspection of the problems in an interactive way
     (transcript-host/register-observer! (make-failure-matcher))

src/automation/dirac/automation/transcript_host.cljs

@@ -6,6 +6,7 @@
                                            get-transcript-label-padding-type]])
   (:require [cljs.core.async :refer [put! <! chan timeout alts! close!]]
             [dirac.automation.transcript :as transcript]
+            [dirac.automation.transcript-streamer :as streamer]
             [oops.core :refer [oget oset! ocall oapply gcall!]]
             [chromex.logging :refer-macros [log warn error info]]
             [cuerdas.core :as cuerdas]
@@ -253,7 +254,8 @@
                     generated-style)]
         (transcript/append-to-transcript! @current-transcript text style)
         (helpers/scroll-page-to-bottom!)
-        (record-output! [effective-label effective-text])))))
+        (record-output! [effective-label effective-text])
+        (streamer/publish! text style)))))
 
 (defn forced-append-to-transcript! [label text & [style]]
   (append-to-transcript! label text style true))
@@ -281,9 +283,10 @@
         (match-observer-record! observer-record value))
       (recur))))
 
-(defn init-transcript! [id normalized?]
+(defn init-transcript! [id normalized? streamer-server-url]
   (let [transcript-el (transcript/create-transcript! (helpers/get-el-by-id id))]
     (reset! current-transcript transcript-el)
     (vreset! normalized-transcript normalized?)
+    (streamer/init! streamer-server-url)
     (run-output-matching-loop!)))
 

src/automation/dirac/automation/transcript_streamer.cljs

@@ -0,0 +1,66 @@
+(ns dirac.automation.transcript-streamer
+  (:require-macros [cljs.core.async.macros :refer [go go-loop]])
+  (:require [cljs.core.async :refer [<! chan sliding-buffer put! timeout]]
+            [chromex.logging :refer-macros [log warn error info]]
+            [dirac.lib.ws-client :as ws-client]))
+
+(defonce current-client (atom nil))
+(defonce transcript-stream (chan (sliding-buffer 1024)))
+
+(defn publish! [text style]
+  (put! transcript-stream {:op    :publish
+                           :text  text
+                           :style style}))
+
+; -- message sending --------------------------------------------------------------------------------------------------------
+
+(defn send! [msg]
+  (if-let [client @current-client]
+    (ws-client/send! client msg)
+    (throw (ex-info "No client!" msg))))
+
+; -- message processing -----------------------------------------------------------------------------------------------------
+
+(defmulti process-message :op)
+
+(defmethod process-message :error [message]
+  (throw (ex-info "Received error message" message))
+  (go
+    {:op      :error
+     :message (:type message)}))
+
+; -- connection -------------------------------------------------------------------------------------------------------------
+
+(defn run-streaming-loop! []
+  (log "transcript-streamer: entering streaming loop...")
+  (go-loop []
+    (when-let [msg (<! transcript-stream)]
+      (send! msg)
+      (recur))
+    (log "transcript-streamer: leaving streaming loop...")))
+
+(defn on-message-handler [_client message]
+  (go
+    (if-let [result (<! (process-message message))]
+      (send! result))))
+
+(defn on-open-handler [client]
+  (run-streaming-loop!))
+
+(defn connect! [server-url opts]
+  (log (str "transcript-streamer: connecting " server-url))
+  (let [default-opts {:name       "Transcript Streamer (client)"
+                      :on-message on-message-handler
+                      :on-open    on-open-handler}
+        effective-opts (merge default-opts opts)
+        client (ws-client/connect! server-url effective-opts)]
+    (reset! current-client client)))
+
+(defn disconnect! []
+  (when-let [client @current-client]
+    (ws-client/close! client)
+    true))
+
+(defn init! [server-url & [opts]]
+  (if (some? server-url)
+    (connect! server-url opts)))

src/logging/dirac/logging.clj

@@ -24,6 +24,9 @@
 (def tests-options
   (standard-layout :magenta))
 
+(def transcript-stream-options
+  (standard-layout :yellow))
+
 (def weasel-options
   (standard-layout :cyan))
 
@@ -36,8 +39,9 @@
       (config/set-loggers!
         :root (merge-options root-options options {:level Level/INFO})                                                        ; root level should be always INFO, otherwise we would get very verbose logs from java libs
         ; test runners
-        "dirac.tests.browser" (merge-options tests-options options)
         "dirac.tests.backend" (merge-options tests-options options)
+        "dirac.tests.browser" (merge-options tests-options options)
+        "dirac.tests.browser.tasks.transcript-streamer-server" (merge-options transcript-stream-options options)
         ; test mocks / helpers
         "dirac.test-lib" (merge-options tests-options options)
         ; agent

src/settings/dirac/settings.clj

@@ -63,6 +63,9 @@
 (def ^:const SIGNAL_SERVER_HOST "localhost")
 (def ^:const SIGNAL_SERVER_PORT 22555)
 
+(def ^:const TRANSCRIPT_STREAMER_SERVER_HOST "localhost")
+(def ^:const TRANSCRIPT_STREAMER_SERVER_PORT 22556)
+
 (def ^:const AUTOMATION_ENTRY_POINT_KEY "diracAutomateDevTools")
 (def ^:const FLUSH_PENDING_FEEDBACK_MESSAGES_KEY "diracFlushPendingFeedbackMessages")
 (def ^:const DIRAC_INTERCOM_KEY "diracIntercom")
@@ -283,3 +286,14 @@
 
 (defmacro get-failed-backend-url-resolution-delay []
   FAILED_BACKEND_URL_RESOLUTION_DELAY)
+
+(defmacro get-transcript-streamer-server-host []
+  TRANSCRIPT_STREAMER_SERVER_HOST)
+
+(defmacro get-transcript-streamer-server-port []
+  TRANSCRIPT_STREAMER_SERVER_PORT)
+
+(defmacro get-transcript-streamer-server-url []
+  (let [port (get-transcript-streamer-server-port)]
+    (str "ws://" (get-transcript-streamer-server-host) (if (some? port) (str ":" port)))))
+

test/browser/src/browser_tests/dirac/tests/browser/runner.clj

@@ -9,11 +9,13 @@
             [dirac.test-lib.nrepl-server :refer [with-nrepl-server]]
             [dirac.test-lib.agent :refer [with-dirac-agent]]
             [dirac.test-lib.taxi :refer [with-taxi-setup]]
-            [dirac.test-lib.nrepl-server :as test-nrepl-server]))
+            [dirac.test-lib.nrepl-server :as test-nrepl-server]
+            [dirac.tests.browser.tasks.transcript-streamer-server :refer [with-transcript-streamer-server]]))
 
 ; this test runner runs tests against real chrome browser using chrome driver
 
-(def setup-fixtures! (join-fixtures [with-fixtures-web-server
+(def setup-fixtures! (join-fixtures [with-transcript-streamer-server
+                                     with-fixtures-web-server
                                      with-nrepl-server
                                      with-dirac-agent
                                      with-browser

test/browser/src/browser_tests/dirac/tests/browser/tasks/transcript_streamer_server.clj

@@ -0,0 +1,37 @@
+(ns dirac.tests.browser.tasks.transcript-streamer-server
+  (:require [clojure.tools.logging :as log]
+            [dirac.settings :refer [get-transcript-streamer-server-host get-transcript-streamer-server-port]]
+            [dirac.lib.ws-server :as ws-server]))
+
+(defonce current-server (volatile! nil))
+
+; -- transcript-streamer server ---------------------------------------------------------------------------------------------
+
+(defn publish! [msg]
+  (let [{:keys [text]} msg]
+    (log/info text)))
+
+(defn on-message-handler [_server _client msg]
+  (log/debug "transcript-streamer server: got transcript-streamer message" msg)
+  (case (:op msg)
+    :ready nil                                                                                                                ; ignore
+    :publish (publish! msg)
+    (log/error "transcript-streamer server: received unrecognized message" msg)))
+
+(defn create-transcript-streamer-server! []
+  (log/debug "transcript-streamer server: creating server")
+  (assert (not @current-server))
+  (vreset! current-server (ws-server/create! {:name       "Transcript Streamer (server)"
+                                              :host       (get-transcript-streamer-server-host)
+                                              :port       (get-transcript-streamer-server-port)
+                                              :on-message on-message-handler})))
+
+(defn destroy-transcript-streamer-server! []
+  (log/debug "transcript-streamer server: destroying server")
+  (assert @current-server)
+  (ws-server/destroy! @current-server))
+
+(defn with-transcript-streamer-server [f]
+  (create-transcript-streamer-server!)
+  (f)
+  (destroy-transcript-streamer-server!))