Skip to content

Commit f4e7469

Browse files
epoberezkinshumvgolove
epoberezkin
and
shumvgolove
authored
xftp server: prometheus metrics (#1595)
* xftp server: prometheus metrics * update Co-authored-by: sh <[email protected]> --------- Co-authored-by: sh <[email protected]>
1 parent 4647d69 commit f4e7469

File tree

8 files changed

+186
-3
lines changed

8 files changed

+186
-3
lines changed

‎simplexmq.cabal‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,7 @@ library
224224
Simplex.FileTransfer.Server.Control
225225
Simplex.FileTransfer.Server.Env
226226
Simplex.FileTransfer.Server.Main
227+
Simplex.FileTransfer.Server.Prometheus
227228
Simplex.FileTransfer.Server.Stats
228229
Simplex.FileTransfer.Server.Store
229230
Simplex.FileTransfer.Server.StoreLog

‎src/Simplex/FileTransfer/Server.hs‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ import Network.Socket
4545
import Simplex.FileTransfer.Protocol
4646
import Simplex.FileTransfer.Server.Control
4747
import Simplex.FileTransfer.Server.Env
48+
import Simplex.FileTransfer.Server.Prometheus
4849
import Simplex.FileTransfer.Server.Stats
4950
import Simplex.FileTransfer.Server.Store
5051
import Simplex.FileTransfer.Server.StoreLog
@@ -69,6 +70,7 @@ import Simplex.Messaging.Transport.HTTP2.Server
6970
import Simplex.Messaging.Transport.Server (runLocalTCPServer)
7071
import Simplex.Messaging.Util
7172
import Simplex.Messaging.Version
73+
import System.Environment (lookupEnv)
7274
import System.Exit (exitFailure)
7375
import System.FilePath ((</>))
7476
import System.IO (hPrint, hPutStrLn, universalNewlineMode)
@@ -105,7 +107,14 @@ xftpServer :: XFTPServerConfig -> TMVar Bool -> M ()
105107
xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpiration, fileExpiration, xftpServerVRange} started = do
106108
mapM_ (expireServerFiles Nothing) fileExpiration
107109
restoreServerStats
108-
raceAny_ (runServer : expireFilesThread_ cfg <> serverStatsThread_ cfg <> controlPortThread_ cfg) `finally` stopServer
110+
raceAny_
111+
( runServer
112+
: expireFilesThread_ cfg
113+
<> serverStatsThread_ cfg
114+
<> prometheusMetricsThread_ cfg
115+
<> controlPortThread_ cfg
116+
)
117+
`finally` stopServer
109118
where
110119
runServer :: M ()
111120
runServer = do
@@ -240,6 +249,30 @@ xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpira
240249
]
241250
liftIO $ threadDelay' interval
242251

252+
prometheusMetricsThread_ :: XFTPServerConfig -> [M ()]
253+
prometheusMetricsThread_ XFTPServerConfig {prometheusInterval = Just interval, prometheusMetricsFile} =
254+
[savePrometheusMetrics interval prometheusMetricsFile]
255+
prometheusMetricsThread_ _ = []
256+
257+
savePrometheusMetrics :: Int -> FilePath -> M ()
258+
savePrometheusMetrics saveInterval metricsFile = do
259+
labelMyThread "savePrometheusMetrics"
260+
liftIO $ putStrLn $ "Prometheus metrics saved every " <> show saveInterval <> " seconds to " <> metricsFile
261+
ss <- asks serverStats
262+
rtsOpts <- liftIO $ maybe ("set " <> rtsOptionsEnv) T.pack <$> lookupEnv (T.unpack rtsOptionsEnv)
263+
let interval = 1000000 * saveInterval
264+
liftIO $ forever $ do
265+
threadDelay interval
266+
ts <- getCurrentTime
267+
sm <- getFileServerMetrics ss rtsOpts
268+
T.writeFile metricsFile $ xftpPrometheusMetrics sm ts
269+
270+
getFileServerMetrics :: FileServerStats -> T.Text -> IO FileServerMetrics
271+
getFileServerMetrics ss rtsOptions = do
272+
d <- getFileServerStatsData ss
273+
let fd = periodStatDataCounts $ _filesDownloaded d
274+
pure FileServerMetrics {statsData = d, filesDownloadedPeriods = fd, rtsOptions}
275+
243276
controlPortThread_ :: XFTPServerConfig -> [M ()]
244277
controlPortThread_ XFTPServerConfig {controlPort = Just port} = [runCPServer port]
245278
controlPortThread_ _ = []

‎src/Simplex/FileTransfer/Server/Env.hs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@ data XFTPServerConfig = XFTPServerConfig
6464
logStatsStartTime :: Int64,
6565
serverStatsLogFile :: FilePath,
6666
serverStatsBackupFile :: Maybe FilePath,
67+
prometheusInterval :: Maybe Int,
68+
prometheusMetricsFile :: FilePath,
6769
transportConfig :: TransportServerConfig,
6870
responseDelay :: Int
6971
}

‎src/Simplex/FileTransfer/Server/Main.hs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ import Simplex.Messaging.Server.Expiration
3030
import Simplex.Messaging.Transport (simplexMQVersion)
3131
import Simplex.Messaging.Transport.Client (TransportHost (..))
3232
import Simplex.Messaging.Transport.Server (ServerCredentials (..), mkTransportServerConfig)
33-
import Simplex.Messaging.Util (safeDecodeUtf8, tshow)
33+
import Simplex.Messaging.Util (eitherToMaybe, safeDecodeUtf8, tshow)
3434
import System.Directory (createDirectoryIfMissing, doesFileExist)
3535
import System.FilePath (combine)
3636
import System.IO (BufferMode (..), hSetBuffering, stderr, stdout)
@@ -89,6 +89,9 @@ xftpServerCLI cfgPath logPath = do
8989
<> "# Expire files after the specified number of hours.\n"
9090
<> ("expire_files_hours: " <> tshow defFileExpirationHours <> "\n\n")
9191
<> "log_stats: off\n\
92+
\\n\
93+
\# Log interval for real-time Prometheus metrics\n\
94+
\# prometheus_interval: 60\n\
9295
\\n\
9396
\[AUTH]\n\
9497
\# Set new_files option to off to completely prohibit uploading new files.\n\
@@ -188,6 +191,8 @@ xftpServerCLI cfgPath logPath = do
188191
logStatsStartTime = 0, -- seconds from 00:00 UTC
189192
serverStatsLogFile = combine logPath "file-server-stats.daily.log",
190193
serverStatsBackupFile = logStats $> combine logPath "file-server-stats.log",
194+
prometheusInterval = eitherToMaybe $ read . T.unpack <$> lookupValue "STORE_LOG" "prometheus_interval" ini,
195+
prometheusMetricsFile = combine logPath "xftp-server-metrics.txt",
191196
transportConfig =
192197
mkTransportServerConfig
193198
(fromMaybe False $ iniOnOff "TRANSPORT" "log_tls_errors" ini)

‎src/Simplex/FileTransfer/Server/Prometheus.hs‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
{-# LANGUAGE DuplicateRecordFields #-}
2+
{-# LANGUAGE NamedFieldPuns #-}
3+
{-# LANGUAGE OverloadedStrings #-}
4+
{-# LANGUAGE TypeApplications #-}
5+
{-# OPTIONS_GHC -fno-warn-unrecognised-pragmas #-}
6+
7+
module Simplex.FileTransfer.Server.Prometheus where
8+
9+
import Data.Int (Int64)
10+
import Data.Text (Text)
11+
import qualified Data.Text as T
12+
import Data.Time.Clock (UTCTime (..), diffUTCTime)
13+
import Data.Time.Clock.System (systemEpochDay)
14+
import Data.Time.Format.ISO8601 (iso8601Show)
15+
import Simplex.FileTransfer.Server.Stats
16+
import Simplex.Messaging.Server.Stats (PeriodStatCounts (..))
17+
import Simplex.Messaging.Transport (simplexMQVersion)
18+
import Simplex.Messaging.Util (tshow)
19+
20+
data FileServerMetrics = FileServerMetrics
21+
{ statsData :: FileServerStatsData,
22+
filesDownloadedPeriods :: PeriodStatCounts,
23+
rtsOptions :: Text
24+
}
25+
26+
rtsOptionsEnv :: Text
27+
rtsOptionsEnv = "XFTP_RTS_OPTIONS"
28+
29+
{-# FOURMOLU_DISABLE\n#-}
30+
xftpPrometheusMetrics :: FileServerMetrics -> UTCTime -> Text
31+
xftpPrometheusMetrics sm ts =
32+
time <> files <> info
33+
where
34+
FileServerMetrics {statsData, filesDownloadedPeriods, rtsOptions} = sm
35+
FileServerStatsData
36+
{ _fromTime,
37+
_filesCreated,
38+
_fileRecipients,
39+
_filesUploaded,
40+
_filesExpired,
41+
_filesDeleted,
42+
_filesBlocked,
43+
_fileDownloads,
44+
_fileDownloadAcks,
45+
_filesCount,
46+
_filesSize
47+
} = statsData
48+
time =
49+
"# Recorded at: " <> T.pack (iso8601Show ts) <> "\n\
50+
\# Stats from: " <> T.pack (iso8601Show _fromTime) <> "\n\
51+
\\n"
52+
files =
53+
"# Files\n\
54+
\# -----\n\
55+
\\n\
56+
\# HELP simplex_xftp_files_created Created files\n\
57+
\# TYPE simplex_xftp_files_created counter\n\
58+
\simplex_xftp_files_created " <> mshow _filesCreated <> "\n\
59+
\# filesCreated\n\
60+
\\n\
61+
\# HELP simplex_xftp_files_recipients Files recipients\n\
62+
\# TYPE simplex_xftp_files_recipients counter\n\
63+
\simplex_xftp_files_recipients " <> mshow _fileRecipients <> "\n\
64+
\# fileRecipients\n\
65+
\\n\
66+
\# HELP simplex_xftp_files_uploaded Uploaded files\n\
67+
\# TYPE simplex_xftp_files_uploaded counter\n\
68+
\simplex_xftp_files_uploaded " <> mshow _filesUploaded <> "\n\
69+
\# filesUploaded\n\
70+
\\n\
71+
\# HELP simplex_xftp_files_expired Expired files\n\
72+
\# TYPE simplex_xftp_files_expired counter\n\
73+
\simplex_xftp_files_expired " <> mshow _filesExpired <> "\n\
74+
\# filesExpired\n\
75+
\\n\
76+
\# HELP simplex_xftp_files_deleted Deleted files\n\
77+
\# TYPE simplex_xftp_files_deleted counter\n\
78+
\simplex_xftp_files_deleted " <> mshow _filesDeleted <> "\n\
79+
\# filesDeleted\n\
80+
\\n\
81+
\# HELP simplex_xftp_files_blocked Blocked files\n\
82+
\# TYPE simplex_xftp_files_blocked counter\n\
83+
\simplex_xftp_files_blocked " <> mshow _filesBlocked <> "\n\
84+
\# filesBlocked\n\
85+
\\n\
86+
\# HELP simplex_xftp_file_downloads File downloads\n\
87+
\# TYPE simplex_xftp_file_downloads counter\n\
88+
\simplex_xftp_file_downloads " <> mshow _fileDownloads <> "\n\
89+
\# fileDownloads\n\
90+
\\n\
91+
\# HELP simplex_xftp_file_download_acks File download ACKs\n\
92+
\# TYPE simplex_xftp_file_download_acks counter\n\
93+
\simplex_xftp_file_download_acks " <> mshow _fileDownloadAcks <> "\n\
94+
\# fileDownloadAcks\n\
95+
\\n\
96+
\# HELP simplex_xftp_files_count_total Total files count \n\
97+
\# TYPE simplex_xftp_files_count_total gauge\n\
98+
\simplex_xftp_files_count_total " <> mshow _filesCount <> "\n\
99+
\# filesCount\n\
100+
\\n\
101+
\# HELP simplex_xftp_files_size Size of files \n\
102+
\# TYPE simplex_xftp_files_size gauge\n\
103+
\simplex_xftp_files_size " <> mshow _filesSize <> "\n\
104+
\# filesSize \n\
105+
\\n\
106+
\# HELP simplex_xftp_files_count_daily Daily files count\n\
107+
\# TYPE simplex_xftp_files_count_daily gauge\n\
108+
\simplex_xftp_files_count_daily " <> mstr (dayCount filesDownloadedPeriods) <> "\n\
109+
\# filesDownloaded.dayCount\n\
110+
\\n\
111+
\# HELP simplex_xftp_files_count_weekly Weekly files count\n\
112+
\# TYPE simplex_xftp_files_count_weekly gauge\n\
113+
\simplex_xftp_files_count_weekly " <> mstr (weekCount filesDownloadedPeriods) <> "\n\
114+
\# filesDownloaded.weekCount\n\
115+
\\n\
116+
\# HELP simplex_xftp_files_count_monthly Monthly files count\n\
117+
\# TYPE simplex_xftp_files_count_monthly gauge\n\
118+
\simplex_xftp_files_count_monthly " <> mstr (monthCount filesDownloadedPeriods) <> "\n\
119+
\# filesDownloaded.monthCount\n\
120+
\\n"
121+
info =
122+
"# Info\n\
123+
\# ----\n\
124+
\\n\
125+
\# HELP simplex_xftp_info Server information. RTS options have to be passed via " <> rtsOptionsEnv <> " env var\n\
126+
\# TYPE simplex_xftp_info gauge\n\
127+
\simplex_xftp_info{version=\"" <> T.pack simplexMQVersion <> "\",rts_options=\"" <> rtsOptions <> "\"} 1\n\
128+
\\n"
129+
mstr a = a <> " " <> tsEpoch
130+
mshow :: Show a => a -> Text
131+
mshow = mstr . tshow
132+
tsEpoch = tshow @Int64 $ floor @Double $ realToFrac (ts `diffUTCTime` epoch) * 1000
133+
epoch = UTCTime systemEpochDay 0
134+
{-# FOURMOLU_ENABLE\n#-}

‎src/Simplex/Messaging/Notifications/Server/Main.hs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,9 @@ ntfServerCLI cfgPath logPath =
167167
<> "Time to retain deleted entities in the database, days.\n"
168168
<> ("# db_deleted_ttl: " <> tshow defaultDeletedTTL <> "\n\n")
169169
<> "log_stats: off\n\n\
170+
\# Log interval for real-time Prometheus metrics\n\
171+
\# prometheus_interval: 60\n\
172+
\\n\
170173
\[AUTH]\n\
171174
\# control_port_admin_password:\n\
172175
\# control_port_user_password:\n\

‎src/Simplex/Messaging/Server/Main/Init.hs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ iniFileContent cfgPath logPath opts host basicAuth controlPortPwds =
9292
<> "# Log daily server statistics to CSV file\n"
9393
<> ("log_stats: " <> onOff logStats <> "\n\n")
9494
<> "# Log interval for real-time Prometheus metrics\n\
95-
\# prometheus_interval: 300\n\n\
95+
\# prometheus_interval: 60\n\n\
9696
\[AUTH]\n\
9797
\# Set new_queues option to off to completely prohibit creating new messaging queues.\n\
9898
\# This can be useful when you want to decommission the server, but not all connections are switched yet.\n\

‎tests/XFTPClient.hs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,9 @@ testXFTPLogFile = "tests/tmp/xftp-server-store.log"
9999
testXFTPStatsBackupFile :: FilePath
100100
testXFTPStatsBackupFile = "tests/tmp/xftp-server-stats.log"
101101

102+
xftpTestPrometheusMetricsFile :: FilePath
103+
xftpTestPrometheusMetricsFile = "tests/tmp/xftp-server-metrics.txt"
104+
102105
testXFTPServerConfig :: XFTPServerConfig
103106
testXFTPServerConfig =
104107
XFTPServerConfig
@@ -127,6 +130,8 @@ testXFTPServerConfig =
127130
logStatsStartTime = 0,
128131
serverStatsLogFile = "tests/tmp/xftp-server-stats.daily.log",
129132
serverStatsBackupFile = Nothing,
133+
prometheusInterval = Nothing,
134+
prometheusMetricsFile = xftpTestPrometheusMetricsFile,
130135
transportConfig = mkTransportServerConfig True (Just alpnSupportedXFTPhandshakes) False,
131136
responseDelay = 0
132137
}

0 commit comments

Comments
 (0)