Skip to content

Commit a3b810e

Browse files
J0sueTMJ0sueTM
committed
feat: send signal to opensearch cluster
1 parent 124d433 commit a3b810e

File tree

5 files changed

+133
-92
lines changed

5 files changed

+133
-92
lines changed

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,5 @@ output.calva-repl
2222
/.clj-kondo
2323
.envrc
2424
redis
25+
localstack
26+

‎deps.edn‎

Lines changed: 44 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,36 @@
11
{:paths ["resources" "src"]
2-
:deps {org.clojure/clojure {:mvn/version "1.11.1"}
3-
com.moclojer/moclojer {:mvn/version "0.3.2"}
4-
com.moclojer/rq {:mvn/version "0.1.4"}
5-
clj-http/clj-http {:mvn/version "3.12.3"}
6-
com.zaxxer/HikariCP {:mvn/version "5.0.1"}
7-
com.stuartsierra/component {:mvn/version "1.1.0"}
8-
com.taoensso/timbre {:mvn/version "6.3.1"}
9-
viesti/timbre-json-appender {:mvn/version "0.2.13"}
10-
com.fzakaria/slf4j-timbre {:mvn/version "0.3.21"}
11-
aero/aero {:mvn/version "1.1.6"}
12-
metosin/muuntaja {:mvn/version "0.6.8"}
13-
metosin/malli {:mvn/version "0.11.0"}
14-
metosin/reitit {:mvn/version "0.5.18"}
15-
metosin/reitit-pedestal {:mvn/version "0.5.18"}
16-
metosin/reitit-swagger {:mvn/version "0.5.18"}
17-
metosin/reitit-swagger-ui {:mvn/version "0.5.18"}
18-
io.sentry/sentry {:mvn/version "7.6.0"}
19-
io.sentry/sentry-clj {:mvn/version "7.6.215"}
20-
com.cognitect.aws/api {:mvn/version "0.8.561"}
21-
com.cognitect.aws/endpoints {:mvn/version "1.1.12.230"}
22-
com.cognitect.aws/s3 {:mvn/version "822.2.1145.0"}
23-
org.postgresql/postgresql {:mvn/version "42.6.0"}
24-
com.github.seancorfield/next.jdbc {:mvn/version "1.3.834"}
25-
io.pedestal/pedestal.jetty {:mvn/version "0.5.10"}
26-
io.pedestal/pedestal.service {:mvn/version "0.5.10"}
27-
commons-io/commons-io {:mvn/version "2.16.1"}
28-
commons-fileupload/commons-fileupload {:mvn/version "1.5"}
29-
migratus/migratus {:mvn/version "1.5.1"}
30-
redis.clients/jedis {:mvn/version "5.1.2"}}
2+
:deps {org.clojure/clojure {:mvn/version "1.11.1"}
3+
com.moclojer/moclojer {:mvn/version "0.3.2"}
4+
com.moclojer/rq {:mvn/version "0.1.4"}
5+
clj-http/clj-http {:mvn/version "3.12.3"}
6+
com.zaxxer/HikariCP {:mvn/version "5.0.1"}
7+
com.stuartsierra/component {:mvn/version "1.1.0"}
8+
com.taoensso/telemere {:mvn/version "1.0.0-beta25"}
9+
com.taoensso/slf4j-telemere {:mvn/version "1.0.0-beta21"}
10+
;; com.taoensso/timbre {:mvn/version "6.3.1"}
11+
;; viesti/timbre-json-appender {:mvn/version "0.2.13"}
12+
;; com.fzakaria/slf4j-timbre {:mvn/version "0.3.21"}
13+
org.clojure/data.json {:mvn/version "2.5.0"}
14+
aero/aero {:mvn/version "1.1.6"}
15+
metosin/muuntaja {:mvn/version "0.6.8"}
16+
metosin/malli {:mvn/version "0.11.0"}
17+
metosin/reitit {:mvn/version "0.5.18"}
18+
metosin/reitit-pedestal {:mvn/version "0.5.18"}
19+
metosin/reitit-swagger {:mvn/version "0.5.18"}
20+
metosin/reitit-swagger-ui {:mvn/version "0.5.18"}
21+
io.sentry/sentry {:mvn/version "7.6.0"}
22+
io.sentry/sentry-clj {:mvn/version "7.6.215"}
23+
com.cognitect.aws/api {:mvn/version "0.8.561"}
24+
com.cognitect.aws/endpoints {:mvn/version "1.1.12.230"}
25+
com.cognitect.aws/s3 {:mvn/version "822.2.1145.0"}
26+
org.postgresql/postgresql {:mvn/version "42.6.0"}
27+
com.github.seancorfield/next.jdbc {:mvn/version "1.3.834"}
28+
io.pedestal/pedestal.jetty {:mvn/version "0.5.10"}
29+
io.pedestal/pedestal.service {:mvn/version "0.5.10"}
30+
commons-io/commons-io {:mvn/version "2.16.1"}
31+
commons-fileupload/commons-fileupload {:mvn/version "1.5"}
32+
migratus/migratus {:mvn/version "1.5.1"}
33+
redis.clients/jedis {:mvn/version "5.1.2"}}
3134
:aliases
3235
{;; clj -A:dev -m com.moclojer.components
3336
:dev
@@ -45,17 +48,17 @@
4548
;; clj -M:test -n com.moclojer.rq.excel-test
4649
:test
4750
{:extra-paths ["test"]
48-
:extra-deps {org.clojars.bigsy/pg-embedded-clj {:mvn/version "1.0.0"}
49-
lambdaisland/kaocha {:mvn/version "1.70.1086"}
50-
lambdaisland/kaocha-cloverage {:mvn/version "1.0.75"}
51-
nubank/matcher-combinators {:mvn/version "3.5.1"}
52-
nubank/state-flow {:mvn/version "5.14.2"}}
53-
:main-opts ["-m" "kaocha.runner" "--no-capture-output"]}
51+
:extra-deps {org.clojars.bigsy/pg-embedded-clj {:mvn/version "1.0.0"}
52+
lambdaisland/kaocha {:mvn/version "1.70.1086"}
53+
lambdaisland/kaocha-cloverage {:mvn/version "1.0.75"}
54+
nubank/matcher-combinators {:mvn/version "3.5.1"}
55+
nubank/state-flow {:mvn/version "5.14.2"}}
56+
:main-opts ["-m" "kaocha.runner" "--no-capture-output"]}
5457

5558
;; clj -M:nrepl
5659
:nrepl
5760
{:extra-deps {cider/cider-nrepl {:mvn/version "0.30.0"}}
58-
:main-opts ["-m" "nrepl.cmdline" "--middleware" "[cider.nrepl/cider-middleware]"]}
61+
:main-opts ["-m" "nrepl.cmdline" "--middleware" "[cider.nrepl/cider-middleware]"]}
5962

6063
;; Lint the source
6164
;; clj -M:lint
@@ -68,8 +71,8 @@
6871
;; clj -X:deploy-clojars
6972
:deploy-clojars
7073
{:extra-deps {slipset/deps-deploy {:mvn/version "RELEASE"}}
71-
:exec-fn deps-deploy.deps-deploy/deploy
72-
:exec-args {:installer :remote
73-
:sign-releases? false
74-
:pom-file "target/classes/META-INF/maven/com.moclojer/components/pom.xml"
75-
:artifact "target/com.moclojer.components.jar"}}}}
74+
:exec-fn deps-deploy.deps-deploy/deploy
75+
:exec-args {:installer :remote
76+
:sign-releases? false
77+
:pom-file "target/classes/META-INF/maven/com.moclojer/components/pom.xml"
78+
:artifact "target/com.moclojer.components.jar"}}}}

‎resources/config.example.edn‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@
2222
:prod "prod"
2323
:default "prod"}
2424
:dns #or [#env SENTRY_AUTH_TOKEN "foobar"]}
25+
:opensearch {:username #or [#env OPENSEARCH_USERNAME "foobar"]
26+
:password #or [#env OPENSEARCH_PASSWORD "foobar"]
27+
:host #or [#env OPENSEARCH_HOST "foobar.com"]
28+
:port #or [#env OPENSEARCH_PORT 00000]}
2529
:mq {:uri #or [#env MQ_URL "redis://localhost:6379"]}
2630
:database {:dbtype "postgres"
2731
:jdbc-url #or [#env DATABASE_URL "postgresql://localhost:5432/postgres?user=postgres&password=postgres"]}}

‎src/com/moclojer/components/core.clj‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,6 @@
2727
[responses]
2828
(http/map->HttpMock {:responses responses}))
2929

30-
(def setup-logger logs/setup)
31-
3230
(defn new-mq
3331
([workers blocking?]
3432
(new-mq workers [] blocking?))

‎src/com/moclojer/components/logs.clj‎

Lines changed: 83 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,53 +1,86 @@
11
(ns com.moclojer.components.logs
22
(:require
3-
[taoensso.timbre :as timbre]
4-
[taoensso.timbre.appenders.core :as core-appenders]
5-
[timbre-json-appender.core :as tas])
6-
(:import
7-
[java.util.logging Filter Handler Logger]))
8-
9-
(defn clean-dep-logs
10-
"clean logs on prod that are not from our application"
11-
[]
12-
(doseq [^Handler handler (.. (Logger/getGlobal)
13-
getParent
14-
getHandlers)]
15-
(.setFilter
16-
handler
17-
(reify Filter
18-
(isLoggable [_ record]
19-
(if-let [^String logger-name (.getLoggerName record)]
20-
(not-any? #(.contains logger-name %)
21-
["jetty" "hikari" "pedestal" "migratus"])
22-
;; returning false explicitly so Java interop doesn't
23-
;; bugout for some reason in the future.
24-
false))))))
25-
26-
(defn clean-timbre-appenders []
27-
(->> (reduce-kv
28-
(fn [acc k _]
29-
(assoc acc k nil))
30-
{} (:appenders timbre/*config*))
31-
(assoc nil :appenders)
32-
timbre/merge-config!))
33-
34-
(defn setup [level stream env]
35-
(clean-timbre-appenders)
36-
(let [prod? (= env :prod)
37-
ns-filter (when prod?
38-
#{"components.*" "back.api.*"
39-
"yaml-generator.*" "cloud-ops.api.*"})
40-
appenders (if prod?
41-
(tas/install)
42-
{:appenders
43-
{:println
44-
(core-appenders/println-appender
45-
{:stream stream})}})]
46-
(when prod? (clean-dep-logs))
47-
(timbre/merge-config!
48-
(merge appenders
49-
{:min-level level
50-
:ns-filter ns-filter}))))
3+
[clojure.data.json :as json]
4+
[clj-http.client :as http-client]
5+
[taoensso.telemere :as t]
6+
[taoensso.telemere.timbre :as timbre])
7+
(:import (java.util.concurrent TimeoutException TimeUnit)))
8+
9+
(defn build-opensearch-base-req
10+
[config]
11+
(let [{:keys [username password host port]} config
12+
url (str "https://" host ":" port "/_bulk")]
13+
{:method :post
14+
:url url
15+
:async? true
16+
:basic-auth [username password]
17+
:content-type :json
18+
:body (json/write-str {:index {:_index "logs"}})}))
19+
20+
(defn ->str-values
21+
[m]
22+
(reduce-kv
23+
(fn [acc k v]
24+
(assoc acc k (cond
25+
(map? v) (->str-values v)
26+
(string? v) (identity v)
27+
:else (pr-str v))))
28+
{} m))
29+
30+
(defn send-opensearch-signal-req
31+
[base-req signal]
32+
(let [log (-> (select-keys signal [:level :ctx :data
33+
:msg_ :error :thread
34+
:uid :inst])
35+
(->str-values)
36+
(json/write-str))
37+
future (http-client/request
38+
(update
39+
base-req :body
40+
str \newline log \newline)
41+
identity #(throw ^Exception %))]
42+
(try
43+
(.get future 1 TimeUnit/SECONDS)
44+
(catch TimeoutException _
45+
(.cancel future true)))))
46+
47+
(defn setup [config level env]
48+
(let [prod? (= env "prod")
49+
os-cfg (when prod? (:opensearch config))
50+
os-base-req (build-opensearch-base-req os-cfg)]
51+
(t/set-min-level! level)
52+
(t/set-ns-filter! {:disallow "*" :allow "com.moclojer.*"})
53+
(when prod?
54+
(telemere/add-signal!
55+
:opensearch
56+
(fn [signal])))))
57+
58+
(comment
59+
(def my-signal
60+
(t/with-signal
61+
(t/log! {:level :info
62+
:data {:hello true
63+
:time "hello world"}}
64+
"hello")))
65+
66+
(def base-req
67+
(build-opensearch-base-req
68+
{:username "foobar"
69+
:password "foobar"
70+
:host "foobar"
71+
:port 25060}))
72+
73+
(send-opensearch-signal-req base-req my-signal)
74+
75+
(http-client/request
76+
(update
77+
base-req :body
78+
str
79+
\newline
80+
(json/write-str my-signal)
81+
\newline))
82+
;;
83+
)
5184

5285
(defmacro log [level & args]
5386
`(timbre/log ~level ~@args))
@@ -56,7 +89,8 @@
5689
{:cid (str "cid-" (random-uuid) "-" (System/currentTimeMillis))})
5790

5891
(comment
59-
(setup [["*" :info]] :auto :prod)
92+
(setup :dev)
93+
6094
(log :info :world)
6195
;;
6296
)

0 commit comments

Comments
 (0)