Skip to content

Commit deeeeea

Browse files
mgmeiermgmeier
authored
Merge pull request #6377 from IntersectMBO/mkarg/new-tracing-changes-maintenance
New tracing: new features and some maintenance
2 parents 0c220b2 + 2c0eb72 commit deeeeea

File tree

49 files changed

+951
-679
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+951
-679
lines changed

‎bench/tx-generator/src/Cardano/Benchmarking/LogTypes.hs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ data TraceBenchTxSubmit txid
123123
| TraceBenchTxSubError Text
124124
| TraceBenchPlutusBudgetSummary PlutusBudgetSummary
125125
-- ^ PlutusBudgetSummary.
126+
| TraceBenchForwardingInterrupted HowToConnect String
126127
deriving stock (Show, Generic)
127128

128129
data SubmissionSummary

‎bench/tx-generator/src/Cardano/Benchmarking/Tracer.hs‎

Lines changed: 44 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
{-# LANGUAGE NamedFieldPuns #-}
1212
{-# LANGUAGE PatternSynonyms #-}
1313
{-# LANGUAGE RankNTypes #-}
14+
{-# LANGUAGE RecursiveDo #-}
1415
{-# LANGUAGE ScopedTypeVariables #-}
1516
{-# LANGUAGE TypeApplications #-}
1617
{-# LANGUAGE TypeOperators #-}
@@ -37,6 +38,7 @@ import Cardano.Node.Startup
3738
import Cardano.Node.Tracing.NodeInfo ()
3839
import Ouroboros.Network.IOManager (IOManager)
3940

41+
import Control.Exception (SomeException (..))
4042
import Control.Monad (forM, guard)
4143
import Data.Aeson as A
4244
import qualified Data.Aeson.KeyMap as KeyMap
@@ -49,7 +51,7 @@ import qualified Data.Text as Text
4951
import Data.Time.Clock
5052
import GHC.Generics
5153

52-
import Trace.Forward.Forwarding (initForwardingDelayed)
54+
import Trace.Forward.Forwarding (InitForwardingConfig (..), initForwardingDelayed)
5355
import Trace.Forward.Utils.TraceObject
5456

5557
pattern TracerNameBench :: Text
@@ -85,9 +87,9 @@ initNullTracers = BenchTracers
8587
-- if the first argument isJust, we assume we have a socket path
8688
-- and want to use trace-dispatcher, so we'll create a forwarding tracer
8789
initTxGenTracers :: Maybe (IOManager, NetworkId, FilePath) -> IO BenchTracers
88-
initTxGenTracers mbForwarding = do
90+
initTxGenTracers mbForwarding = mdo
8991
mbStdoutTracer <- fmap Just standardTracer
90-
mbForwardingTracer <- prepareForwardingTracer
92+
mbForwardingTracer <- prepareForwardingTracer tracers
9193
confState <- emptyConfigReflection
9294

9395
let
@@ -108,21 +110,34 @@ initTxGenTracers mbForwarding = do
108110
connectTracer <- mkTracer TracerNameConnect mbStdoutTracer mbForwardingTracer
109111
submitTracer <- mkTracer TracerNameSubmit mbStdoutTracer mbForwardingTracer
110112

111-
traceWith benchTracer (TraceTxGeneratorVersion Version.txGeneratorVersion)
112-
113-
return $ BenchTracers
114-
{ btTxSubmit_ = benchTracer
115-
, btConnect_ = connectTracer
116-
, btSubmission2_ = submitTracer
117-
, btN2N_ = n2nSubmitTracer
118-
}
113+
let
114+
tracers = BenchTracers
115+
{ btTxSubmit_ = benchTracer
116+
, btConnect_ = connectTracer
117+
, btSubmission2_ = submitTracer
118+
, btN2N_ = n2nSubmitTracer
119+
}
120+
121+
traceWith (btTxSubmit_ tracers) (TraceTxGeneratorVersion Version.txGeneratorVersion)
122+
return tracers
119123
where
120-
prepareForwardingTracer :: IO (Maybe (Trace IO FormattedMessage))
121-
prepareForwardingTracer = forM mbForwarding $
124+
prepareForwardingTracer :: BenchTracers -> IO (Maybe (Trace IO FormattedMessage))
125+
prepareForwardingTracer benchTracer = forM mbForwarding $
122126
\(iomgr, networkId, tracerSocket) -> do
123-
let forwardingConf = fromMaybe defaultForwarder (tcForwarder initialTraceConfig)
127+
let
128+
forwardingConf = fromMaybe defaultForwarder (tcForwarder initialTraceConfig)
129+
howToConnect = Net.LocalPipe tracerSocket
130+
initForwConf = InitForwardingWith
131+
{ initNetworkMagic = toNetworkMagic networkId
132+
, initEKGStore = Nothing
133+
, initHowToConnect = howToConnect
134+
, initForwarderMode = Initiator
135+
, initOnForwardInterruption = Just $ \(SomeException e) ->
136+
traceWith (btTxSubmit_ benchTracer) (TraceBenchForwardingInterrupted howToConnect $ show e)
137+
, initOnQueueOverflow = Nothing
138+
}
124139
(forwardSink, dpStore, kickoffForwarder) <-
125-
initForwardingDelayed iomgr forwardingConf (toNetworkMagic networkId) Nothing $ Just (Net.LocalPipe tracerSocket, Initiator)
140+
initForwardingDelayed iomgr forwardingConf initForwConf
126141

127142
-- we need to provide NodeInfo DataPoint, to forward generator's name
128143
-- to the acceptor application (for example, 'cardano-tracer').
@@ -205,8 +220,14 @@ instance (ConstructorsOf f, ConstructorsOf g) => ConstructorsOf (f :+: g) where
205220
instance (Constructor ('MetaCons n f r)) => ConstructorsOf (C1 ('MetaCons n f r) x) where
206221
constructorsOf _ = [ conName @('MetaCons n f r) undefined ]
207222

223+
208224
instance LogFormatting (TraceBenchTxSubmit TxId) where
209-
forHuman = Text.pack . show
225+
forHuman = \case
226+
TraceBenchForwardingInterrupted howToConnect errMsg ->
227+
Text.pack $ "trace forwarding connection with " <> show howToConnect <> " failed: " <> errMsg
228+
_
229+
-> ""
230+
210231
forMachine DMinimal _ = mempty
211232
forMachine DNormal t = mconcat [ "kind" .= A.String (genericName t) ]
212233
forMachine DDetailed t = forMachine DMaximum t
@@ -276,6 +297,11 @@ instance LogFormatting (TraceBenchTxSubmit TxId) where
276297
mconcat [ "kind" .= A.String "TraceBenchPlutusBudgetSummary"
277298
, "summary" .= toJSON summary
278299
]
300+
TraceBenchForwardingInterrupted howToConnect msg ->
301+
mconcat [ "kind" .= A.String "TraceBenchForwardingInterrupted"
302+
, "conn" .= howToConnect
303+
, "msg" .= msg
304+
]
279305

280306
instance MetaTrace (TraceBenchTxSubmit TxId) where
281307
namespaceFor TraceTxGeneratorVersion {} = Namespace [] ["TxGeneratorVersion"]
@@ -295,6 +321,7 @@ instance MetaTrace (TraceBenchTxSubmit TxId) where
295321
namespaceFor TraceBenchTxSubDebug {} = Namespace [] ["BenchTxSubDebug"]
296322
namespaceFor TraceBenchTxSubError {} = Namespace [] ["BenchTxSubError"]
297323
namespaceFor TraceBenchPlutusBudgetSummary {} = Namespace [] ["BenchPlutusBudgetSummary"]
324+
namespaceFor TraceBenchForwardingInterrupted {} = Namespace [] ["ForwardingInterrupted"]
298325

299326
severityFor _ _ = Just Info
300327

@@ -318,6 +345,7 @@ instance MetaTrace (TraceBenchTxSubmit TxId) where
318345
, Namespace [] ["BenchTxSubDebug"]
319346
, Namespace [] ["BenchTxSubError"]
320347
, Namespace [] ["BenchPlutusBudgetSummary"]
348+
, Namespace [] ["ForwardingInterrupted"]
321349
]
322350

323351
instance LogFormatting NodeToNodeSubmissionTrace where

‎cardano-node/cardano-node.cabal‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -211,8 +211,8 @@ library
211211
, sop-extras
212212
, text >= 2.0
213213
, time
214-
, trace-dispatcher ^>= 2.10.0
215-
, trace-forward ^>= 2.3.0
214+
, trace-dispatcher ^>= 2.11.0
215+
, trace-forward ^>= 2.4.0
216216
, trace-resources ^>= 0.2.4
217217
, tracer-transformers
218218
, transformers

‎cardano-node/src/Cardano/Node/Tracing/API.hs‎

Lines changed: 30 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
{-# LANGUAGE MonoLocalBinds #-}
33
{-# LANGUAGE PackageImports #-}
44
{-# LANGUAGE RankNTypes #-}
5+
{-# LANGUAGE RecordWildCards #-}
6+
{-# LANGUAGE RecursiveDo #-}
57
{-# LANGUAGE ScopedTypeVariables #-}
68

79
module Cardano.Node.Tracing.API
@@ -36,6 +38,7 @@ import Ouroboros.Network.NodeToNode (RemoteAddress)
3638
import Prelude
3739

3840
import Control.DeepSeq (deepseq)
41+
import Control.Exception (SomeException (..))
3942
import Control.Monad (forM_)
4043
import "contra-tracer" Control.Tracer (traceWith)
4144
import "trace-dispatcher" Control.Tracer (nullTracer)
@@ -46,7 +49,7 @@ import Network.Mux.Trace (TraceLabelPeer (..))
4649
import Network.Socket (HostName)
4750
import System.Metrics as EKG
4851

49-
import Trace.Forward.Forwarding (initForwardingDelayed)
52+
import Trace.Forward.Forwarding (InitForwardingConfig (..), initForwardingDelayed)
5053
import Trace.Forward.Utils.TraceObject (writeToSink)
5154

5255

@@ -111,7 +114,7 @@ initTraceDispatcher nc p networkMagic nodeKernel noBlockForging = do
111114
, IO (Maybe String)
112115
, Tracers RemoteAddress LocalAddress blk IO
113116
)
114-
mkTracers trConfig = do
117+
mkTracers trConfig = mdo
115118
ekgStore <- EKG.newStore
116119
EKG.registerGcMetrics ekgStore
117120
ekgTrace <- ekgTracer trConfig ekgStore
@@ -126,28 +129,39 @@ initTraceDispatcher nc p networkMagic nodeKernel noBlockForging = do
126129
if forwarderBackendEnabled
127130
then do
128131
-- TODO: check if this is the correct way to use withIOManager
129-
(forwardSink, dpStore, kickoffForwarder) <- withIOManager $ \iomgr -> do
130-
let tracerSocketMode :: Maybe (HowToConnect, ForwarderMode)
131-
tracerSocketMode = ncTraceForwardSocket nc
132+
(forwardSink, dpStore, kickoffForwarder') <- withIOManager $ \iomgr ->
133+
let initForwConf :: InitForwardingConfig
134+
initForwConf = case ncTraceForwardSocket nc of
135+
Nothing -> InitForwardingNone
136+
Just (initHowToConnect, initForwarderMode) ->
137+
InitForwardingWith
138+
{ initNetworkMagic = networkMagic
139+
, initEKGStore = Just ekgStore
140+
, initOnForwardInterruption = Just $ \(SomeException e) ->
141+
traceWith (nodeStateTracer tracers) (NodeTracingForwardingInterrupted initHowToConnect $ show e)
142+
, initOnQueueOverflow = Nothing
143+
, ..
144+
}
132145

133146
forwardingConf :: TraceOptionForwarder
134147
forwardingConf = fromMaybe defaultForwarder (tcForwarder trConfig)
135-
initForwardingDelayed iomgr forwardingConf networkMagic (Just ekgStore) tracerSocketMode
136-
pure (forwardTracer (writeToSink forwardSink), dataPointTracer dpStore, kickoffForwarder)
148+
in initForwardingDelayed iomgr forwardingConf initForwConf
149+
150+
pure (forwardTracer (writeToSink forwardSink), dataPointTracer dpStore, kickoffForwarder')
137151
else
138152
-- Since 'Forwarder' backend isn't enabled, there is no forwarding.
139153
-- So we use nullTracers to ignore 'TraceObject's and 'DataPoint's.
140154
pure (Trace nullTracer, Trace nullTracer, pure ())
141155

142-
(,,) kickoffForwarder kickoffPrometheusSimple
143-
<$> mkDispatchTracers
144-
nodeKernel
145-
stdoutTrace
146-
fwdTracer
147-
(Just ekgTrace)
148-
dpTracer
149-
trConfig
150-
p
156+
tracers <- mkDispatchTracers
157+
nodeKernel
158+
stdoutTrace
159+
fwdTracer
160+
(Just ekgTrace)
161+
dpTracer
162+
trConfig
163+
p
164+
pure (kickoffForwarder, kickoffPrometheusSimple, tracers)
151165

152166
where
153167
-- This backend can only be used globally, i.e. will always apply to the namespace root.

‎cardano-node/src/Cardano/Node/Tracing/StateRep.hs‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ deriving instance (NFData StartupState)
9999
data NodeState
100100
= NodeTracingOnlineConfiguring
101101
| NodeTracingFailure String
102+
| NodeTracingForwardingInterrupted HowToConnect String
102103
| NodeOpeningDbs OpeningDbs
103104
| NodeReplays Replays
104105
| NodeInitChainSelection InitChainSelection
@@ -130,16 +131,27 @@ instance LogFormatting NodeState where
130131
[ "kind" .= String "NodeShutdown", "shutdown" .= toJSON x]
131132
NodeTracingFailure x -> mconcat
132133
[ "kind" .= String "NodeTracingFailure", "message" .= toJSON x]
133-
134-
forHuman (NodeTracingFailure errMsg) = T.pack errMsg
135-
forHuman _ = ""
136-
134+
NodeTracingForwardingInterrupted howToConnect x -> mconcat
135+
[ "kind" .= String "NodeTracingForwardingInterrupted"
136+
, "conn" .= howToConnect
137+
, "message" .= toJSON x
138+
]
139+
140+
forHuman = \case
141+
NodeTracingFailure errMsg ->
142+
T.pack errMsg
143+
NodeTracingForwardingInterrupted howToConnect errMsg ->
144+
T.pack $ "trace forwarding connection with " <> show howToConnect <> " failed: " <> errMsg
145+
_
146+
-> ""
137147

138148
instance MetaTrace NodeState where
139149
namespaceFor NodeTracingOnlineConfiguring {} =
140150
Namespace [] ["NodeTracingOnlineConfiguring"]
141-
namespaceFor NodeTracingFailure {} =
151+
namespaceFor NodeTracingFailure {} =
142152
Namespace [] ["NodeTracingFailure"]
153+
namespaceFor NodeTracingForwardingInterrupted {} =
154+
Namespace [] ["NodeTracingForwardingInterrupted"]
143155
namespaceFor NodeOpeningDbs {} =
144156
Namespace [] ["OpeningDbs"]
145157
namespaceFor NodeReplays {} =
@@ -159,6 +171,8 @@ instance MetaTrace NodeState where
159171
Just Info
160172
severityFor (Namespace _ ["NodeTracingFailure"]) _ =
161173
Just Error
174+
severityFor (Namespace _ ["NodeTracingForwardingInterrupted"]) _ =
175+
Just Warning
162176
severityFor (Namespace _ ["OpeningDbs"]) _ =
163177
Just Info
164178
severityFor (Namespace _ ["NodeReplays"]) _ =
@@ -180,6 +194,8 @@ instance MetaTrace NodeState where
180194
"Tracing system came online, system configuring now"
181195
documentFor (Namespace _ ["NodeTracingFailure"]) = Just
182196
"Tracing system experienced a non-fatal failure during startup"
197+
documentFor (Namespace _ ["NodeTracingForwardingInterrupted"]) = Just
198+
"Trace/metrics forwarding connection was interrupted"
183199
documentFor (Namespace _ ["OpeningDbs"]) = Just
184200
"ChainDB components being opened"
185201
documentFor (Namespace _ ["NodeReplays"]) = Just
@@ -199,6 +215,7 @@ instance MetaTrace NodeState where
199215
allNamespaces = [
200216
Namespace [] ["NodeTracingOnlineConfiguring"]
201217
, Namespace [] ["NodeTracingFailure"]
218+
, Namespace [] ["NodeTracingForwardingInterrupted"]
202219
, Namespace [] ["OpeningDbs"]
203220
, Namespace [] ["NodeReplays"]
204221
, Namespace [] ["NodeInitChainSelection"]

‎cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -88,16 +88,16 @@ instance ( LogFormatting (Header blk)
8888
) => LogFormatting (ChainDB.TraceEvent blk) where
8989
forHuman ChainDB.TraceLastShutdownUnclean =
9090
"ChainDB is not clean. Validating all immutable chunks"
91-
forHuman (ChainDB.TraceAddBlockEvent v) = forHumanOrMachine v
92-
forHuman (ChainDB.TraceFollowerEvent v) = forHumanOrMachine v
93-
forHuman (ChainDB.TraceCopyToImmutableDBEvent v) = forHumanOrMachine v
94-
forHuman (ChainDB.TraceGCEvent v) = forHumanOrMachine v
95-
forHuman (ChainDB.TraceInitChainSelEvent v) = forHumanOrMachine v
96-
forHuman (ChainDB.TraceOpenEvent v) = forHumanOrMachine v
97-
forHuman (ChainDB.TraceIteratorEvent v) = forHumanOrMachine v
98-
forHuman (ChainDB.TraceLedgerDBEvent v) = forHumanOrMachine v
99-
forHuman (ChainDB.TraceImmutableDBEvent v) = forHumanOrMachine v
100-
forHuman (ChainDB.TraceVolatileDBEvent v) = forHumanOrMachine v
91+
forHuman (ChainDB.TraceAddBlockEvent v) = forHuman v
92+
forHuman (ChainDB.TraceFollowerEvent v) = forHuman v
93+
forHuman (ChainDB.TraceCopyToImmutableDBEvent v) = forHuman v
94+
forHuman (ChainDB.TraceGCEvent v) = forHuman v
95+
forHuman (ChainDB.TraceInitChainSelEvent v) = forHuman v
96+
forHuman (ChainDB.TraceOpenEvent v) = forHuman v
97+
forHuman (ChainDB.TraceIteratorEvent v) = forHuman v
98+
forHuman (ChainDB.TraceLedgerDBEvent v) = forHuman v
99+
forHuman (ChainDB.TraceImmutableDBEvent v) = forHuman v
100+
forHuman (ChainDB.TraceVolatileDBEvent v) = forHuman v
101101
forHuman (ChainDB.TraceChainSelStarvationEvent ev) = case ev of
102102
ChainDB.ChainSelStarvation RisingEdge ->
103103
"Chain Selection was starved."
@@ -429,12 +429,12 @@ instance ( LogFormatting (Header blk)
429429
forHuman (ChainDB.SwitchedToAFork es _ _ c) =
430430
"Switched to a fork, new tip: " <> renderPointAsPhrase (AF.headPoint c) <>
431431
Text.concat [ "\nEvent: " <> showT e | e <- es ]
432-
forHuman (ChainDB.AddBlockValidation ev') = forHumanOrMachine ev'
432+
forHuman (ChainDB.AddBlockValidation ev') = forHuman ev'
433433
forHuman (ChainDB.AddedBlockToVolatileDB pt _ _ enclosing) =
434434
case enclosing of
435435
RisingEdge -> "Chain about to add block " <> renderRealPointAsPhrase pt
436436
FallingEdge -> "Chain added block " <> renderRealPointAsPhrase pt
437-
forHuman (ChainDB.PipeliningEvent ev') = forHumanOrMachine ev'
437+
forHuman (ChainDB.PipeliningEvent ev') = forHuman ev'
438438
forHuman (ChainDB.AddedReprocessLoEBlocksToQueue edgeSz) =
439439
case edgeSz of
440440
RisingEdge ->
@@ -1068,7 +1068,7 @@ instance MetaTrace (ChainDB.TraceGCEvent blk) where
10681068

10691069
instance (ConvertRawHash blk, LedgerSupportsProtocol blk)
10701070
=> LogFormatting (ChainDB.TraceInitChainSelEvent blk) where
1071-
forHuman (ChainDB.InitChainSelValidation v) = forHumanOrMachine v
1071+
forHuman (ChainDB.InitChainSelValidation v) = forHuman v
10721072
forHuman ChainDB.InitialChainSelected{} =
10731073
"Initial chain selected"
10741074
forHuman ChainDB.StartedInitChainSelection {} =
@@ -1336,7 +1336,7 @@ instance MetaTrace (ChainDB.TraceOpenEvent blk) where
13361336
instance ( StandardHash blk
13371337
, ConvertRawHash blk
13381338
) => LogFormatting (ChainDB.TraceIteratorEvent blk) where
1339-
forHuman (ChainDB.UnknownRangeRequested ev') = forHumanOrMachine ev'
1339+
forHuman (ChainDB.UnknownRangeRequested ev') = forHuman ev'
13401340
forHuman (ChainDB.BlockMissingFromVolatileDB realPt) = mconcat
13411341
[ "This block is no longer in the VolatileDB because it has been garbage"
13421342
, " collected. It might now be in the ImmDB if it was part of the"
@@ -2848,8 +2848,8 @@ instance ( LogFormatting (LedgerError blk)
28482848
forMachine dtal (ExtValidationErrorLedger err) = forMachine dtal err
28492849
forMachine dtal (ExtValidationErrorHeader err) = forMachine dtal err
28502850

2851-
forHuman (ExtValidationErrorLedger err) = forHumanOrMachine err
2852-
forHuman (ExtValidationErrorHeader err) = forHumanOrMachine err
2851+
forHuman (ExtValidationErrorLedger err) = forHuman err
2852+
forHuman (ExtValidationErrorHeader err) = forHuman err
28532853

28542854
asMetrics (ExtValidationErrorLedger err) = asMetrics err
28552855
asMetrics (ExtValidationErrorHeader err) = asMetrics err

0 commit comments

Comments
 (0)