diff --git a/bench/tx-generator/src/Cardano/Benchmarking/LogTypes.hs b/bench/tx-generator/src/Cardano/Benchmarking/LogTypes.hs index 81d9f5166ff..32eb5c295dd 100644 --- a/bench/tx-generator/src/Cardano/Benchmarking/LogTypes.hs +++ b/bench/tx-generator/src/Cardano/Benchmarking/LogTypes.hs @@ -123,6 +123,7 @@ data TraceBenchTxSubmit txid | TraceBenchTxSubError Text | TraceBenchPlutusBudgetSummary PlutusBudgetSummary -- ^ PlutusBudgetSummary. + | TraceBenchForwardingInterrupted HowToConnect String deriving stock (Show, Generic) data SubmissionSummary diff --git a/bench/tx-generator/src/Cardano/Benchmarking/Tracer.hs b/bench/tx-generator/src/Cardano/Benchmarking/Tracer.hs index b6ddce84de0..dc5d59f7a8f 100644 --- a/bench/tx-generator/src/Cardano/Benchmarking/Tracer.hs +++ b/bench/tx-generator/src/Cardano/Benchmarking/Tracer.hs @@ -11,6 +11,7 @@ {-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE PatternSynonyms #-} {-# LANGUAGE RankNTypes #-} +{-# LANGUAGE RecursiveDo #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} {-# LANGUAGE TypeOperators #-} @@ -37,6 +38,7 @@ import Cardano.Node.Startup import Cardano.Node.Tracing.NodeInfo () import Ouroboros.Network.IOManager (IOManager) +import Control.Exception (SomeException (..)) import Control.Monad (forM, guard) import Data.Aeson as A import qualified Data.Aeson.KeyMap as KeyMap @@ -49,7 +51,7 @@ import qualified Data.Text as Text import Data.Time.Clock import GHC.Generics -import Trace.Forward.Forwarding (initForwardingDelayed) +import Trace.Forward.Forwarding (InitForwardingConfig (..), initForwardingDelayed) import Trace.Forward.Utils.TraceObject pattern TracerNameBench :: Text @@ -85,9 +87,9 @@ initNullTracers = BenchTracers -- if the first argument isJust, we assume we have a socket path -- and want to use trace-dispatcher, so we'll create a forwarding tracer initTxGenTracers :: Maybe (IOManager, NetworkId, FilePath) -> IO BenchTracers -initTxGenTracers mbForwarding = do +initTxGenTracers mbForwarding = mdo mbStdoutTracer <- fmap Just standardTracer - mbForwardingTracer <- prepareForwardingTracer + mbForwardingTracer <- prepareForwardingTracer tracers confState <- emptyConfigReflection let @@ -108,21 +110,34 @@ initTxGenTracers mbForwarding = do connectTracer <- mkTracer TracerNameConnect mbStdoutTracer mbForwardingTracer submitTracer <- mkTracer TracerNameSubmit mbStdoutTracer mbForwardingTracer - traceWith benchTracer (TraceTxGeneratorVersion Version.txGeneratorVersion) - - return $ BenchTracers - { btTxSubmit_ = benchTracer - , btConnect_ = connectTracer - , btSubmission2_ = submitTracer - , btN2N_ = n2nSubmitTracer - } + let + tracers = BenchTracers + { btTxSubmit_ = benchTracer + , btConnect_ = connectTracer + , btSubmission2_ = submitTracer + , btN2N_ = n2nSubmitTracer + } + + traceWith (btTxSubmit_ tracers) (TraceTxGeneratorVersion Version.txGeneratorVersion) + return tracers where - prepareForwardingTracer :: IO (Maybe (Trace IO FormattedMessage)) - prepareForwardingTracer = forM mbForwarding $ + prepareForwardingTracer :: BenchTracers -> IO (Maybe (Trace IO FormattedMessage)) + prepareForwardingTracer benchTracer = forM mbForwarding $ \(iomgr, networkId, tracerSocket) -> do - let forwardingConf = fromMaybe defaultForwarder (tcForwarder initialTraceConfig) + let + forwardingConf = fromMaybe defaultForwarder (tcForwarder initialTraceConfig) + howToConnect = Net.LocalPipe tracerSocket + initForwConf = InitForwardingWith + { initNetworkMagic = toNetworkMagic networkId + , initEKGStore = Nothing + , initHowToConnect = howToConnect + , initForwarderMode = Initiator + , initOnForwardInterruption = Just $ \(SomeException e) -> + traceWith (btTxSubmit_ benchTracer) (TraceBenchForwardingInterrupted howToConnect $ show e) + , initOnQueueOverflow = Nothing + } (forwardSink, dpStore, kickoffForwarder) <- - initForwardingDelayed iomgr forwardingConf (toNetworkMagic networkId) Nothing $ Just (Net.LocalPipe tracerSocket, Initiator) + initForwardingDelayed iomgr forwardingConf initForwConf -- we need to provide NodeInfo DataPoint, to forward generator's name -- to the acceptor application (for example, 'cardano-tracer'). @@ -205,8 +220,14 @@ instance (ConstructorsOf f, ConstructorsOf g) => ConstructorsOf (f :+: g) where instance (Constructor ('MetaCons n f r)) => ConstructorsOf (C1 ('MetaCons n f r) x) where constructorsOf _ = [ conName @('MetaCons n f r) undefined ] + instance LogFormatting (TraceBenchTxSubmit TxId) where - forHuman = Text.pack . show + forHuman = \case + TraceBenchForwardingInterrupted howToConnect errMsg -> + Text.pack $ "trace forwarding connection with " <> show howToConnect <> " failed: " <> errMsg + _ + -> "" + forMachine DMinimal _ = mempty forMachine DNormal t = mconcat [ "kind" .= A.String (genericName t) ] forMachine DDetailed t = forMachine DMaximum t @@ -276,6 +297,11 @@ instance LogFormatting (TraceBenchTxSubmit TxId) where mconcat [ "kind" .= A.String "TraceBenchPlutusBudgetSummary" , "summary" .= toJSON summary ] + TraceBenchForwardingInterrupted howToConnect msg -> + mconcat [ "kind" .= A.String "TraceBenchForwardingInterrupted" + , "conn" .= howToConnect + , "msg" .= msg + ] instance MetaTrace (TraceBenchTxSubmit TxId) where namespaceFor TraceTxGeneratorVersion {} = Namespace [] ["TxGeneratorVersion"] @@ -295,6 +321,7 @@ instance MetaTrace (TraceBenchTxSubmit TxId) where namespaceFor TraceBenchTxSubDebug {} = Namespace [] ["BenchTxSubDebug"] namespaceFor TraceBenchTxSubError {} = Namespace [] ["BenchTxSubError"] namespaceFor TraceBenchPlutusBudgetSummary {} = Namespace [] ["BenchPlutusBudgetSummary"] + namespaceFor TraceBenchForwardingInterrupted {} = Namespace [] ["ForwardingInterrupted"] severityFor _ _ = Just Info @@ -318,6 +345,7 @@ instance MetaTrace (TraceBenchTxSubmit TxId) where , Namespace [] ["BenchTxSubDebug"] , Namespace [] ["BenchTxSubError"] , Namespace [] ["BenchPlutusBudgetSummary"] + , Namespace [] ["ForwardingInterrupted"] ] instance LogFormatting NodeToNodeSubmissionTrace where diff --git a/cardano-node/cardano-node.cabal b/cardano-node/cardano-node.cabal index cc1024e54dd..1de0bc2aaf0 100644 --- a/cardano-node/cardano-node.cabal +++ b/cardano-node/cardano-node.cabal @@ -211,8 +211,8 @@ library , sop-extras , text >= 2.0 , time - , trace-dispatcher ^>= 2.10.0 - , trace-forward ^>= 2.3.0 + , trace-dispatcher ^>= 2.11.0 + , trace-forward ^>= 2.4.0 , trace-resources ^>= 0.2.4 , tracer-transformers , transformers diff --git a/cardano-node/src/Cardano/Node/Tracing/API.hs b/cardano-node/src/Cardano/Node/Tracing/API.hs index d18ee73e9f7..81e1bd82dfc 100644 --- a/cardano-node/src/Cardano/Node/Tracing/API.hs +++ b/cardano-node/src/Cardano/Node/Tracing/API.hs @@ -2,6 +2,8 @@ {-# LANGUAGE MonoLocalBinds #-} {-# LANGUAGE PackageImports #-} {-# LANGUAGE RankNTypes #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE RecursiveDo #-} {-# LANGUAGE ScopedTypeVariables #-} module Cardano.Node.Tracing.API @@ -36,6 +38,7 @@ import Ouroboros.Network.NodeToNode (RemoteAddress) import Prelude import Control.DeepSeq (deepseq) +import Control.Exception (SomeException (..)) import Control.Monad (forM_) import "contra-tracer" Control.Tracer (traceWith) import "trace-dispatcher" Control.Tracer (nullTracer) @@ -46,7 +49,7 @@ import Network.Mux.Trace (TraceLabelPeer (..)) import Network.Socket (HostName) import System.Metrics as EKG -import Trace.Forward.Forwarding (initForwardingDelayed) +import Trace.Forward.Forwarding (InitForwardingConfig (..), initForwardingDelayed) import Trace.Forward.Utils.TraceObject (writeToSink) @@ -111,7 +114,7 @@ initTraceDispatcher nc p networkMagic nodeKernel noBlockForging = do , IO (Maybe String) , Tracers RemoteAddress LocalAddress blk IO ) - mkTracers trConfig = do + mkTracers trConfig = mdo ekgStore <- EKG.newStore EKG.registerGcMetrics ekgStore ekgTrace <- ekgTracer trConfig ekgStore @@ -126,28 +129,39 @@ initTraceDispatcher nc p networkMagic nodeKernel noBlockForging = do if forwarderBackendEnabled then do -- TODO: check if this is the correct way to use withIOManager - (forwardSink, dpStore, kickoffForwarder) <- withIOManager $ \iomgr -> do - let tracerSocketMode :: Maybe (HowToConnect, ForwarderMode) - tracerSocketMode = ncTraceForwardSocket nc + (forwardSink, dpStore, kickoffForwarder') <- withIOManager $ \iomgr -> + let initForwConf :: InitForwardingConfig + initForwConf = case ncTraceForwardSocket nc of + Nothing -> InitForwardingNone + Just (initHowToConnect, initForwarderMode) -> + InitForwardingWith + { initNetworkMagic = networkMagic + , initEKGStore = Just ekgStore + , initOnForwardInterruption = Just $ \(SomeException e) -> + traceWith (nodeStateTracer tracers) (NodeTracingForwardingInterrupted initHowToConnect $ show e) + , initOnQueueOverflow = Nothing + , .. + } forwardingConf :: TraceOptionForwarder forwardingConf = fromMaybe defaultForwarder (tcForwarder trConfig) - initForwardingDelayed iomgr forwardingConf networkMagic (Just ekgStore) tracerSocketMode - pure (forwardTracer (writeToSink forwardSink), dataPointTracer dpStore, kickoffForwarder) + in initForwardingDelayed iomgr forwardingConf initForwConf + + pure (forwardTracer (writeToSink forwardSink), dataPointTracer dpStore, kickoffForwarder') else -- Since 'Forwarder' backend isn't enabled, there is no forwarding. -- So we use nullTracers to ignore 'TraceObject's and 'DataPoint's. pure (Trace nullTracer, Trace nullTracer, pure ()) - (,,) kickoffForwarder kickoffPrometheusSimple - <$> mkDispatchTracers - nodeKernel - stdoutTrace - fwdTracer - (Just ekgTrace) - dpTracer - trConfig - p + tracers <- mkDispatchTracers + nodeKernel + stdoutTrace + fwdTracer + (Just ekgTrace) + dpTracer + trConfig + p + pure (kickoffForwarder, kickoffPrometheusSimple, tracers) where -- This backend can only be used globally, i.e. will always apply to the namespace root. diff --git a/cardano-node/src/Cardano/Node/Tracing/StateRep.hs b/cardano-node/src/Cardano/Node/Tracing/StateRep.hs index 7b65e9e17ab..0a4e12bea43 100644 --- a/cardano-node/src/Cardano/Node/Tracing/StateRep.hs +++ b/cardano-node/src/Cardano/Node/Tracing/StateRep.hs @@ -99,6 +99,7 @@ deriving instance (NFData StartupState) data NodeState = NodeTracingOnlineConfiguring | NodeTracingFailure String + | NodeTracingForwardingInterrupted HowToConnect String | NodeOpeningDbs OpeningDbs | NodeReplays Replays | NodeInitChainSelection InitChainSelection @@ -130,16 +131,27 @@ instance LogFormatting NodeState where [ "kind" .= String "NodeShutdown", "shutdown" .= toJSON x] NodeTracingFailure x -> mconcat [ "kind" .= String "NodeTracingFailure", "message" .= toJSON x] - - forHuman (NodeTracingFailure errMsg) = T.pack errMsg - forHuman _ = "" - + NodeTracingForwardingInterrupted howToConnect x -> mconcat + [ "kind" .= String "NodeTracingForwardingInterrupted" + , "conn" .= howToConnect + , "message" .= toJSON x + ] + + forHuman = \case + NodeTracingFailure errMsg -> + T.pack errMsg + NodeTracingForwardingInterrupted howToConnect errMsg -> + T.pack $ "trace forwarding connection with " <> show howToConnect <> " failed: " <> errMsg + _ + -> "" instance MetaTrace NodeState where namespaceFor NodeTracingOnlineConfiguring {} = Namespace [] ["NodeTracingOnlineConfiguring"] - namespaceFor NodeTracingFailure {} = + namespaceFor NodeTracingFailure {} = Namespace [] ["NodeTracingFailure"] + namespaceFor NodeTracingForwardingInterrupted {} = + Namespace [] ["NodeTracingForwardingInterrupted"] namespaceFor NodeOpeningDbs {} = Namespace [] ["OpeningDbs"] namespaceFor NodeReplays {} = @@ -159,6 +171,8 @@ instance MetaTrace NodeState where Just Info severityFor (Namespace _ ["NodeTracingFailure"]) _ = Just Error + severityFor (Namespace _ ["NodeTracingForwardingInterrupted"]) _ = + Just Warning severityFor (Namespace _ ["OpeningDbs"]) _ = Just Info severityFor (Namespace _ ["NodeReplays"]) _ = @@ -180,6 +194,8 @@ instance MetaTrace NodeState where "Tracing system came online, system configuring now" documentFor (Namespace _ ["NodeTracingFailure"]) = Just "Tracing system experienced a non-fatal failure during startup" + documentFor (Namespace _ ["NodeTracingForwardingInterrupted"]) = Just + "Trace/metrics forwarding connection was interrupted" documentFor (Namespace _ ["OpeningDbs"]) = Just "ChainDB components being opened" documentFor (Namespace _ ["NodeReplays"]) = Just @@ -199,6 +215,7 @@ instance MetaTrace NodeState where allNamespaces = [ Namespace [] ["NodeTracingOnlineConfiguring"] , Namespace [] ["NodeTracingFailure"] + , Namespace [] ["NodeTracingForwardingInterrupted"] , Namespace [] ["OpeningDbs"] , Namespace [] ["NodeReplays"] , Namespace [] ["NodeInitChainSelection"] diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs index e41a61bdf98..a8b5f32dcf5 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs @@ -88,16 +88,16 @@ instance ( LogFormatting (Header blk) ) => LogFormatting (ChainDB.TraceEvent blk) where forHuman ChainDB.TraceLastShutdownUnclean = "ChainDB is not clean. Validating all immutable chunks" - forHuman (ChainDB.TraceAddBlockEvent v) = forHumanOrMachine v - forHuman (ChainDB.TraceFollowerEvent v) = forHumanOrMachine v - forHuman (ChainDB.TraceCopyToImmutableDBEvent v) = forHumanOrMachine v - forHuman (ChainDB.TraceGCEvent v) = forHumanOrMachine v - forHuman (ChainDB.TraceInitChainSelEvent v) = forHumanOrMachine v - forHuman (ChainDB.TraceOpenEvent v) = forHumanOrMachine v - forHuman (ChainDB.TraceIteratorEvent v) = forHumanOrMachine v - forHuman (ChainDB.TraceLedgerDBEvent v) = forHumanOrMachine v - forHuman (ChainDB.TraceImmutableDBEvent v) = forHumanOrMachine v - forHuman (ChainDB.TraceVolatileDBEvent v) = forHumanOrMachine v + forHuman (ChainDB.TraceAddBlockEvent v) = forHuman v + forHuman (ChainDB.TraceFollowerEvent v) = forHuman v + forHuman (ChainDB.TraceCopyToImmutableDBEvent v) = forHuman v + forHuman (ChainDB.TraceGCEvent v) = forHuman v + forHuman (ChainDB.TraceInitChainSelEvent v) = forHuman v + forHuman (ChainDB.TraceOpenEvent v) = forHuman v + forHuman (ChainDB.TraceIteratorEvent v) = forHuman v + forHuman (ChainDB.TraceLedgerDBEvent v) = forHuman v + forHuman (ChainDB.TraceImmutableDBEvent v) = forHuman v + forHuman (ChainDB.TraceVolatileDBEvent v) = forHuman v forHuman (ChainDB.TraceChainSelStarvationEvent ev) = case ev of ChainDB.ChainSelStarvation RisingEdge -> "Chain Selection was starved." @@ -429,12 +429,12 @@ instance ( LogFormatting (Header blk) forHuman (ChainDB.SwitchedToAFork es _ _ c) = "Switched to a fork, new tip: " <> renderPointAsPhrase (AF.headPoint c) <> Text.concat [ "\nEvent: " <> showT e | e <- es ] - forHuman (ChainDB.AddBlockValidation ev') = forHumanOrMachine ev' + forHuman (ChainDB.AddBlockValidation ev') = forHuman ev' forHuman (ChainDB.AddedBlockToVolatileDB pt _ _ enclosing) = case enclosing of RisingEdge -> "Chain about to add block " <> renderRealPointAsPhrase pt FallingEdge -> "Chain added block " <> renderRealPointAsPhrase pt - forHuman (ChainDB.PipeliningEvent ev') = forHumanOrMachine ev' + forHuman (ChainDB.PipeliningEvent ev') = forHuman ev' forHuman (ChainDB.AddedReprocessLoEBlocksToQueue edgeSz) = case edgeSz of RisingEdge -> @@ -1068,7 +1068,7 @@ instance MetaTrace (ChainDB.TraceGCEvent blk) where instance (ConvertRawHash blk, LedgerSupportsProtocol blk) => LogFormatting (ChainDB.TraceInitChainSelEvent blk) where - forHuman (ChainDB.InitChainSelValidation v) = forHumanOrMachine v + forHuman (ChainDB.InitChainSelValidation v) = forHuman v forHuman ChainDB.InitialChainSelected{} = "Initial chain selected" forHuman ChainDB.StartedInitChainSelection {} = @@ -1336,7 +1336,7 @@ instance MetaTrace (ChainDB.TraceOpenEvent blk) where instance ( StandardHash blk , ConvertRawHash blk ) => LogFormatting (ChainDB.TraceIteratorEvent blk) where - forHuman (ChainDB.UnknownRangeRequested ev') = forHumanOrMachine ev' + forHuman (ChainDB.UnknownRangeRequested ev') = forHuman ev' forHuman (ChainDB.BlockMissingFromVolatileDB realPt) = mconcat [ "This block is no longer in the VolatileDB because it has been garbage" , " collected. It might now be in the ImmDB if it was part of the" @@ -2848,8 +2848,8 @@ instance ( LogFormatting (LedgerError blk) forMachine dtal (ExtValidationErrorLedger err) = forMachine dtal err forMachine dtal (ExtValidationErrorHeader err) = forMachine dtal err - forHuman (ExtValidationErrorLedger err) = forHumanOrMachine err - forHuman (ExtValidationErrorHeader err) = forHumanOrMachine err + forHuman (ExtValidationErrorLedger err) = forHuman err + forHuman (ExtValidationErrorHeader err) = forHuman err asMetrics (ExtValidationErrorLedger err) = asMetrics err asMetrics (ExtValidationErrorHeader err) = asMetrics err diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/Consensus.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/Consensus.hs index 5d76470c82a..c204e8d418a 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/Consensus.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/Consensus.hs @@ -102,7 +102,7 @@ instance LogFormatting a => LogFormatting (TraceLabelCreds a) where mconcat $ ("credentials" .= toJSON creds) : [forMachine dtal a] forHuman (TraceLabelCreds creds a) = - "With label " <> (Text.pack . show) creds <> ", " <> forHumanOrMachine a + "With label " <> (Text.pack . show) creds <> ", " <> forHuman a asMetrics (TraceLabelCreds _creds a) = asMetrics a @@ -131,7 +131,7 @@ instance (LogFormatting peer, Show peer, LogFormatting a) forMachine dtal (TraceLabelPeer peerid a) = mconcat [ "peer" .= forMachine dtal peerid ] <> forMachine dtal a forHuman (TraceLabelPeer peerid a) = "Peer is: (" <> showT peerid - <> "). " <> forHumanOrMachine a + <> "). " <> forHuman a asMetrics (TraceLabelPeer _peerid a) = asMetrics a instance MetaTrace a => MetaTrace (TraceLabelPeer label a) where @@ -1006,8 +1006,6 @@ instance ( LogFormatting peer , "peers" .= toJSON (map (forMachine dtal) (toList peers)) ] - forHuman = forHumanFromMachine - instance MetaTrace (TraceGDDEvent peer blk) where namespaceFor _ = Namespace [] ["TraceGDDEvent"] @@ -1032,8 +1030,6 @@ instance ( HasHeader blk , "idling" .= toJSON idling ] - forHuman = forHumanFromMachine - -------------------------------------------------------------------------------- -- SanityCheckIssue Tracer @@ -2209,8 +2205,6 @@ instance ( LogFormatting peer, Show peer, ConvertRawHash blk BecauseCsjDisengage -> String "BecauseCsjDisengage" BecauseCsjDisconnect -> String "BecauseCsjDisconnect" - forHuman = forHumanFromMachine - instance MetaTrace (Jumping.TraceEventCsj peer blk) where namespaceFor = \case BecomingObjector{} -> Namespace [] ["BecomingObjector"] diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/Diffusion.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/Diffusion.hs index ec55570c197..512aa7dc088 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/Diffusion.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/Diffusion.hs @@ -45,8 +45,8 @@ instance (LogFormatting peer, LogFormatting tr, Typeable tr) => mconcat [ "kind" .= (show . typeOf $ ev) , "bearer" .= forMachine dtal b , "event" .= forMachine dtal ev ] - forHuman (Mux.WithBearer b ev) = "With mux bearer " <> forHumanOrMachine b - <> ". " <> forHumanOrMachine ev + forHuman (Mux.WithBearer b ev) = "With mux bearer " <> forHuman b + <> ". " <> forHuman ev instance MetaTrace tr => MetaTrace (Mux.WithBearer peer tr) where namespaceFor (Mux.WithBearer _peer obj) = (nsCast . namespaceFor) obj diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/NodeToClient.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/NodeToClient.hs index a504b19c4a9..0a6dc16da24 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/NodeToClient.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/NodeToClient.hs @@ -33,8 +33,8 @@ instance LogFormatting (Simple.AnyMessage ps) forMachine dtal (Simple.TraceRecvMsg m) = mconcat [ "kind" .= String "Recv" , "msg" .= forMachine dtal m ] - forHuman (Simple.TraceSendMsg m) = "Send: " <> forHumanOrMachine m - forHuman (Simple.TraceRecvMsg m) = "Receive: " <> forHumanOrMachine m + forHuman (Simple.TraceSendMsg m) = "Send: " <> forHuman m + forHuman (Simple.TraceRecvMsg m) = "Receive: " <> forHuman m asMetrics (Simple.TraceSendMsg m) = asMetrics m asMetrics (Simple.TraceRecvMsg m) = asMetrics m @@ -46,8 +46,8 @@ instance LogFormatting (Stateful.AnyMessage ps f) forMachine dtal (Stateful.TraceRecvMsg m) = mconcat [ "kind" .= String "Recv" , "msg" .= forMachine dtal m ] - forHuman (Stateful.TraceSendMsg m) = "Send: " <> forHumanOrMachine m - forHuman (Stateful.TraceRecvMsg m) = "Receive: " <> forHumanOrMachine m + forHuman (Stateful.TraceSendMsg m) = "Send: " <> forHuman m + forHuman (Stateful.TraceRecvMsg m) = "Receive: " <> forHuman m asMetrics (Stateful.TraceSendMsg m) = asMetrics m asMetrics (Stateful.TraceRecvMsg m) = asMetrics m diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/P2P.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/P2P.hs index aa2046108cf..96539155e56 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/P2P.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/P2P.hs @@ -987,7 +987,6 @@ instance LogFormatting (PeerSelectionCounters (Cardano.ExtraPeerSelectionSetsWit , "activeBootstrapPeers" .= snd (Cardano.viewActiveBootstrapPeers extraCounters) , "ActiveBootstrapPeersDemotions" .= snd (Cardano.viewActiveBootstrapPeersDemotions extraCounters) ] - forHuman = forHumanFromMachine asMetrics psc = case psc of PeerSelectionCountersHWC {..} -> @@ -1577,7 +1576,9 @@ instance (Show peerAddr, ToJSON peerAddr) , "from" .= toJSON (ConnectionManager.fromState tr) , "to" .= toJSON (ConnectionManager.toState tr) ] + forHuman = pack . show + asMetrics _ = [] instance MetaTrace (ConnectionManager.AbstractTransitionTrace peerAddr) where diff --git a/cardano-tracer/CHANGELOG.md b/cardano-tracer/CHANGELOG.md index 136f79eb365..fc6c3145e3c 100644 --- a/cardano-tracer/CHANGELOG.md +++ b/cardano-tracer/CHANGELOG.md @@ -1,5 +1,14 @@ # ChangeLog +## 0.3.6 (November 2025) +* Implement Prometheus HTTP service discovery (SD) under the URL `/targets` +* Add optional config field `"prometheusLabels": { "": "", ... }` for custom labels to be attached with Prometheus SD +* Use `TracerTrace.forMachine` directly instead of going via derived `TracerTrace.toJSON`; remove unused `TracerTrace` JSON instances +* Use proper 'camelCase' for machine-readable `TracerTrace` +* Proper tracing (vs. dumping to stdout) for `showProblemIfAny` and for forwarding connection interruptions +* Remove redundant `runInLoop` in favour of `trace-dispatcher`'s implementation +* Split up journal handler implementation into internal modules `Systemd` and `NoSystemd` (maintenance) + ## 0.3.5 (October, 2025) * Updated to `ekg-forward-1.0`, `ouroboros-network-0.22.3`, `ouroboros-network-api-0.16` and `ouroboros-network-0.22.3`. * Updated metric names diff --git a/cardano-tracer/bench/cardano-tracer-bench.hs b/cardano-tracer/bench/cardano-tracer-bench.hs index 67193e0d9f3..b2e4a29c063 100644 --- a/cardano-tracer/bench/cardano-tracer-bench.hs +++ b/cardano-tracer/bench/cardano-tracer-bench.hs @@ -155,6 +155,7 @@ main = do , hasForwarding = Nothing , resourceFreq = Nothing , ekgRequestFull = Nothing + , prometheusLabels = Nothing } generate :: Int -> IO [TraceObject] diff --git a/cardano-tracer/cardano-tracer.cabal b/cardano-tracer/cardano-tracer.cabal index 2c64dfd1e52..fd7e123f45a 100644 --- a/cardano-tracer/cardano-tracer.cabal +++ b/cardano-tracer/cardano-tracer.cabal @@ -1,7 +1,7 @@ cabal-version: 3.0 name: cardano-tracer -version: 0.3.5 +version: 0.3.6 synopsis: A service for logging and monitoring over Cardano nodes description: A service for logging and monitoring over Cardano nodes. category: Cardano, @@ -149,8 +149,10 @@ library Cardano.Tracer.Types Cardano.Tracer.Utils - other-modules: Cardano.Tracer.Handlers.Notifications.Timer + other-modules: Cardano.Tracer.Handlers.Logs.Journal.NoSystemd + Cardano.Tracer.Handlers.Notifications.Timer Paths_cardano_tracer + autogen-modules: Paths_cardano_tracer if flag(rtview) @@ -163,6 +165,7 @@ library if flag(systemd) && os(linux) build-depends: libsystemd-journal >= 1.4.4 + other-modules: Cardano.Tracer.Handlers.Logs.Journal.Systemd if (flag(systemd) && os(linux)) || flag(rtview) build-depends: unordered-containers @@ -197,8 +200,8 @@ library , string-qq , text , time - , trace-dispatcher ^>= 2.10.0 - , trace-forward ^>= 2.3.0 + , trace-dispatcher ^>= 2.11.0 + , trace-forward ^>= 2.4.0 , trace-resources ^>= 0.2.4 , wai ^>= 3.2 , warp ^>= 3.4 diff --git a/cardano-tracer/configuration/metrics_help.json b/cardano-tracer/configuration/metrics_help.json index c635b2579dd..ba7420f6c13 100644 --- a/cardano-tracer/configuration/metrics_help.json +++ b/cardano-tracer/configuration/metrics_help.json @@ -35,6 +35,7 @@ "forgedSlotLast": "Slot number of the last forged block", "forging_enabled": "A node without forger credentials or started as non-producing has forging disabled.", "forks": "counter for forks", + "GSM.state": "The state of the Genesis State Machine. 0 = PreSyncing, 1 = Syncing, 2 = CaughtUp.", "haskell_compiler_major": "Cardano compiler version information", "haskell_compiler_minor": "Cardano compiler version information", "haskell_compiler_patch": "Cardano compiler version information", diff --git a/cardano-tracer/docs/cardano-tracer.md b/cardano-tracer/docs/cardano-tracer.md index 8d18bb86392..51c2876c42c 100644 --- a/cardano-tracer/docs/cardano-tracer.md +++ b/cardano-tracer/docs/cardano-tracer.md @@ -19,6 +19,7 @@ - [Logging](#logging) - [Logs Rotation](#logs-rotation) - [Prometheus](#prometheus) + - [Prometheus HTTP service discovery](#prometheus-http-sd) - [EKG Monitoring](#ekg-monitoring) - [Verbosity](#verbosity) - [RTView](#rtview) @@ -462,7 +463,28 @@ or } ``` +### Prometheus HTTP service discovery +The `/targets` path can be used for Prometheus HTTP service discovery. This lets +Prometheus dynamically discover all connected nodes, and scrape their metrics. +Below is a minimal example of a corresponding job definition that goes into the +`prometheus.yml` configuration: + +```yaml + - job_name: "cardano-tracer" + http_sd_configs: + - url: 'http://127.0.0.1:3200/targets' # <-- Your cardano-tracer's real hostname:prometheus port +``` + +Each target will have a label `node_name` which corresponds to the `TraceOptionNodeName` setting in the respective node config. + +In `cardano-tracer`'s config, you can optionally provide additional labels to be attached to *all* targets +(default is no additional labels): +```json + "prometheusLabels": { + "": "", ... + } +``` ## EKG Monitoring diff --git a/cardano-tracer/src/Cardano/Tracer/Acceptors/Run.hs b/cardano-tracer/src/Cardano/Tracer/Acceptors/Run.hs index 0c7574f2a26..028b6833f88 100644 --- a/cardano-tracer/src/Cardano/Tracer/Acceptors/Run.hs +++ b/cardano-tracer/src/Cardano/Tracer/Acceptors/Run.hs @@ -5,16 +5,16 @@ module Cardano.Tracer.Acceptors.Run ( runAcceptors ) where +import Cardano.Logging.Types (TraceObject) +import Cardano.Logging.Utils (runInLoop) import Cardano.Tracer.Acceptors.Client import Cardano.Tracer.Acceptors.Server import Cardano.Tracer.Configuration import Cardano.Tracer.Environment import Cardano.Tracer.MetaTrace -import Cardano.Tracer.Utils -import Cardano.Logging.Types (TraceObject) -import qualified Cardano.Logging.Types as Net import Control.Concurrent.Async (forConcurrently_) +import Control.Exception (SomeException (..)) import "contra-tracer" Control.Tracer (Tracer, contramap, nullTracer, stdoutTracer) import qualified Data.List.NonEmpty as NE import Data.Maybe (fromMaybe) @@ -39,15 +39,19 @@ runAcceptors tracerEnv@TracerEnv{teTracer} tracerEnvRTView = do AcceptAt howToConnect -> -- Run one server that accepts connections from the nodes. runInLoop - (runAcceptorsServer tracerEnv tracerEnvRTView howToConnect $ acceptorsConfigs (Net.howToConnectString howToConnect)) - verbosity howToConnect initialPauseInSec + (runAcceptorsServer tracerEnv tracerEnvRTView howToConnect $ acceptorsConfigs (show howToConnect)) + (handleOnInterruption howToConnect) initialPauseInSec 10 ConnectTo localSocks -> -- Run N clients that initiate connections to the nodes. forConcurrently_ (NE.nub localSocks) \howToConnect -> runInLoop - (runAcceptorsClient tracerEnv tracerEnvRTView howToConnect $ acceptorsConfigs (Net.howToConnectString howToConnect)) - verbosity howToConnect initialPauseInSec + (runAcceptorsClient tracerEnv tracerEnvRTView howToConnect $ acceptorsConfigs (show howToConnect)) + (handleOnInterruption howToConnect) initialPauseInSec 30 where + handleOnInterruption howToConnect (SomeException e) + | verbosity == Just Minimum = pure () + | otherwise = traceWith teTracer $ TracerForwardingInterrupted howToConnect $ show e + TracerConfig{network, ekgRequestFreq, verbosity, ekgRequestFull} = teConfig tracerEnv ekgUseFullRequests = fromMaybe False ekgRequestFull diff --git a/cardano-tracer/src/Cardano/Tracer/Acceptors/Server.hs b/cardano-tracer/src/Cardano/Tracer/Acceptors/Server.hs index e30de1fd18b..37c0470c7e2 100644 --- a/cardano-tracer/src/Cardano/Tracer/Acceptors/Server.hs +++ b/cardano-tracer/src/Cardano/Tracer/Acceptors/Server.hs @@ -64,7 +64,7 @@ runAcceptorsServer -> IO () runAcceptorsServer tracerEnv tracerEnvRTView howToConnect ( ekgConfig, tfConfig, dpfConfig) = withIOManager \iocp -> do - traceWith (teTracer tracerEnv) $ TracerSockListen (Net.howToConnectString howToConnect) + traceWith (teTracer tracerEnv) $ TracerSockListen (show howToConnect) case howToConnect of Net.LocalPipe p -> doListenToForwarderLocal diff --git a/cardano-tracer/src/Cardano/Tracer/Configuration.hs b/cardano-tracer/src/Cardano/Tracer/Configuration.hs index 3e7baff6115..a166364b7ff 100644 --- a/cardano-tracer/src/Cardano/Tracer/Configuration.hs +++ b/cardano-tracer/src/Cardano/Tracer/Configuration.hs @@ -2,7 +2,6 @@ {-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE DuplicateRecordFields #-} {-# LANGUAGE LambdaCase #-} -{-# LANGUAGE MultiWayIf #-} {-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RecordWildCards #-} @@ -11,7 +10,7 @@ {- HLINT ignore "Use any" -} module Cardano.Tracer.Configuration - ( Address -- (..) + ( Address , Net.HowToConnect (..) , Endpoint (..) , setEndpoint @@ -24,14 +23,14 @@ module Cardano.Tracer.Configuration , TracerConfig (..) , Verbosity (..) , readTracerConfig - , parseHostPort ) where +import Cardano.Logging.Types (HowToConnect) import qualified Cardano.Logging.Types as Log +import qualified Cardano.Logging.Types as Net import Control.Applicative ((<|>)) -import Data.Aeson (FromJSON (..), ToJSON (..), withText, withObject, (.:)) -import Data.Aeson.Types (Parser, Value) +import Data.Aeson (FromJSON (..), ToJSON (..), withObject, (.:)) import Data.Fixed (Pico) import Data.Function ((&)) import Data.Functor ((<&>)) @@ -45,52 +44,15 @@ import Data.Maybe (catMaybes) import Data.String (fromString) import Data.Text (Text) import qualified Data.Text as Text -import qualified Data.Text.Read as Text import Data.Word (Word16, Word32, Word64) import Data.Yaml (decodeFileEither) import GHC.Generics (Generic) import Network.Wai.Handler.Warp (HostPreference, Port, Settings, setHost, setPort) import System.Exit (die) -import Cardano.Logging.Types (HowToConnect) -import qualified Cardano.Logging.Types as Net - type Address :: Type type Address = HowToConnect --- first try to host:port, and if that fails revert to parsing any --- string literal and assume it is a localpipe. -instance FromJSON HowToConnect where - parseJSON :: Value -> Parser HowToConnect - parseJSON = withText "HowToConnect" $ \t -> - (uncurry Net.RemoteSocket <$> parseHostPort t) - <|> ( Net.LocalPipe <$> parseLocalPipe t) - -instance ToJSON HowToConnect where - toJSON :: HowToConnect -> Value - toJSON = toJSON . Net.howToConnectString - -parseLocalPipe :: Text -> Parser FilePath -parseLocalPipe t - | Text.null t = fail "parseLocalPipe: empty Text" - | otherwise = pure $ Text.unpack t - -parseHostPort :: Text -> Parser (Text, Word16) -parseHostPort t - | Text.null t - = fail "parseHostPort: empty Text" - | otherwise - = let - (host_, portText) = Text.breakOnEnd ":" t - host = maybe "" fst (Text.unsnoc host_) - in if - | Text.null host -> fail "parseHostPort: Empty host or no colon found." - | Text.null portText -> fail "parseHostPort: Empty port." - | Right (port, remainder) <- Text.decimal portText - , Text.null remainder - , 0 <= port, port <= 65535 -> pure (host, port) - | otherwise -> fail "parseHostPort: Non-numeric port or value out of range." - -- | Endpoint for internal services. data Endpoint = Endpoint { epHost :: !String @@ -196,13 +158,14 @@ data TracerConfig = TracerConfig , Maybe [[Text]] , Log.TraceOptionForwarder )) - , logging :: !(NonEmpty LoggingParams) -- ^ Logging parameters. - , rotation :: !(Maybe RotationParams) -- ^ Rotation parameters. - , verbosity :: !(Maybe Verbosity) -- ^ Verbosity of the tracer itself. - , metricsNoSuffix :: !(Maybe Bool) -- ^ Prometheus ONLY: Dropping metrics name suffixes (like "_int") increases similiarity with old system names - if desired; default: False - , metricsHelp :: !(Maybe FileOrMap) -- ^ Prometheus ONLY: JSON file or object containing a key-value map "metric name -> help text" for "# HELP " annotations - , resourceFreq :: !(Maybe Int) -- ^ Frequency (1/millisecond) for gathering resource data. - , ekgRequestFull :: !(Maybe Bool) -- ^ Request full set of metrics always, vs. deltas only (safer, but more overhead); default: False + , logging :: !(NonEmpty LoggingParams) -- ^ Logging parameters. + , rotation :: !(Maybe RotationParams) -- ^ Rotation parameters. + , verbosity :: !(Maybe Verbosity) -- ^ Verbosity of the tracer itself. + , metricsNoSuffix :: !(Maybe Bool) -- ^ Prometheus ONLY: Dropping metrics name suffixes (like "_int") increases similiarity with old system names - if desired; default: False + , metricsHelp :: !(Maybe FileOrMap) -- ^ Prometheus ONLY: JSON file or object containing a key-value map "metric name -> help text" for "# HELP " annotations + , resourceFreq :: !(Maybe Int) -- ^ Frequency (1/millisecond) for gathering resource data. + , ekgRequestFull :: !(Maybe Bool) -- ^ Request full set of metrics always, vs. deltas only (safer, but more overhead); default: False + , prometheusLabels :: !(Maybe (Map Text Text)) -- ^ A common label set for all Prometheus scrape targets (only used in Prometheus HTTP service discovery) } deriving stock (Eq, Show, Generic) deriving anyclass (FromJSON, ToJSON) diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/Journal.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/Journal.hs index df2f2608c18..152e1c09d88 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/Journal.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/Journal.hs @@ -1,71 +1,11 @@ {-# LANGUAGE CPP #-} -#ifdef SYSTEMD -{-# LANGUAGE NamedFieldPuns #-} -{-# LANGUAGE OverloadedStrings #-} -#endif module Cardano.Tracer.Handlers.Logs.Journal - ( writeTraceObjectsToJournal + ( module Impl ) where #ifdef SYSTEMD -import qualified Cardano.Logging as L -#endif -import Cardano.Logging (TraceObject (..)) -import Cardano.Tracer.Configuration (LogFormat (..)) -import Cardano.Tracer.Types (NodeName) - -#ifdef SYSTEMD -import Data.Char (isDigit) -import Data.Maybe (fromMaybe) -import qualified Data.HashMap.Strict as HM -import qualified Data.Text as T -import Data.Text.Encoding (encodeUtf8) -import Data.Time.Format (defaultTimeLocale, formatTime) - -import Systemd.Journal (Priority (..), message, mkJournalField, priority, - sendJournalFields, syslogIdentifier) - --- | Store 'TraceObject's in Linux systemd's journal service. -writeTraceObjectsToJournal :: LogFormat -> NodeName -> [TraceObject] -> IO () -writeTraceObjectsToJournal logFormat nodeName = - mapM_ (sendJournalFields . mkJournalFields) - where - -- when no forHuman message is implemented for a trace, fallback to forMachine (same as for file handler) - getMsg :: TraceObject -> T.Text - getMsg = case logFormat of - ForMachine -> toMachine - ForHuman -> \TraceObject{toHuman, toMachine} -> fromMaybe toMachine toHuman - - mkJournalFields trObj@TraceObject{toSeverity, toNamespace, toThreadId, toTimestamp} = - syslogIdentifier nodeName - <> message (getMsg trObj) - <> priority (mkPriority toSeverity) - <> HM.fromList - [ (namespace, encodeUtf8 $ mkName toNamespace) - , (thread, encodeUtf8 $ T.filter isDigit toThreadId) - , (time, encodeUtf8 $ formatAsIso8601 toTimestamp) - ] - - mkName [] = "noname" - mkName names = T.intercalate "." names - - namespace = mkJournalField "namespace" - thread = mkJournalField "thread" - time = mkJournalField "time" - - formatAsIso8601 = T.pack . formatTime defaultTimeLocale "%F %T%12QZ" - - mkPriority L.Debug = Debug - mkPriority L.Info = Info - mkPriority L.Notice = Notice - mkPriority L.Warning = Warning - mkPriority L.Error = Error - mkPriority L.Critical = Critical - mkPriority L.Alert = Alert - mkPriority L.Emergency = Emergency +import Cardano.Tracer.Handlers.Logs.Journal.Systemd as Impl #else --- It works only on Linux distributions with systemd support. -writeTraceObjectsToJournal :: LogFormat -> NodeName -> [TraceObject] -> IO () -writeTraceObjectsToJournal _ _ _ = pure () +import Cardano.Tracer.Handlers.Logs.Journal.NoSystemd as Impl #endif diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/Journal/NoSystemd.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/Journal/NoSystemd.hs new file mode 100644 index 00000000000..bf0a985e763 --- /dev/null +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/Journal/NoSystemd.hs @@ -0,0 +1,11 @@ +module Cardano.Tracer.Handlers.Logs.Journal.NoSystemd + ( writeTraceObjectsToJournal + ) where + +import Cardano.Logging (TraceObject) +import Cardano.Tracer.Configuration (LogFormat) +import Cardano.Tracer.Types (NodeName) + + +writeTraceObjectsToJournal :: LogFormat -> NodeName -> [TraceObject] -> IO () +writeTraceObjectsToJournal _ _ _ = pure () diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/Journal/Systemd.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/Journal/Systemd.hs new file mode 100644 index 00000000000..17f42e6f1c2 --- /dev/null +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/Journal/Systemd.hs @@ -0,0 +1,62 @@ +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ViewPatterns #-} + +module Cardano.Tracer.Handlers.Logs.Journal.Systemd + ( writeTraceObjectsToJournal + ) where + +import Cardano.Logging (TraceObject (..)) +import qualified Cardano.Logging as L +import Cardano.Tracer.Configuration (LogFormat (..)) +import Cardano.Tracer.Handlers.Utils (normalizeNamespace) +import Cardano.Tracer.Types (NodeName) + +import Data.Char (isDigit) +import qualified Data.HashMap.Strict as HM +import Data.Maybe (fromMaybe) +import qualified Data.Text as T +import Data.Text.Encoding (encodeUtf8) +import Data.Time.Format (defaultTimeLocale, formatTime) + +import Systemd.Journal (Priority (..), message, mkJournalField, priority, + sendJournalFields, syslogIdentifier) + + +-- | Store 'TraceObject's in Linux systemd's journal service. +writeTraceObjectsToJournal :: LogFormat -> NodeName -> [TraceObject] -> IO () +writeTraceObjectsToJournal logFormat nodeName = + mapM_ (sendJournalFields . mkJournalFields) + where + -- when no forHuman message is implemented for a trace, fallback to forMachine (same as for file handler) + getMsg :: TraceObject -> T.Text + getMsg = case logFormat of + ForMachine -> toMachine + ForHuman -> \TraceObject{toHuman, toMachine} -> fromMaybe toMachine toHuman + + mkJournalFields trObj@TraceObject{toSeverity, toNamespace, toThreadId, toTimestamp} = + syslogIdentifier nodeName + <> message (getMsg trObj) + <> priority (mkPriority toSeverity) + <> HM.fromList + [ (namespace, encodeUtf8 $ mkName toNamespace) + , (thread, encodeUtf8 $ T.filter isDigit toThreadId) + , (time, encodeUtf8 $ formatAsIso8601 toTimestamp) + ] + + mkName (normalizeNamespace -> ns) = if T.null ns then "noname" else ns + + namespace = mkJournalField "namespace" + thread = mkJournalField "thread" + time = mkJournalField "time" + + formatAsIso8601 = T.pack . formatTime defaultTimeLocale "%F %T%12QZ" + + mkPriority L.Debug = Debug + mkPriority L.Info = Info + mkPriority L.Notice = Notice + mkPriority L.Warning = Warning + mkPriority L.Error = Error + mkPriority L.Critical = Critical + mkPriority L.Alert = Alert + mkPriority L.Emergency = Emergency diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/Rotator.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/Rotator.hs index 4e63baf87ed..a8661dcd204 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/Rotator.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/Rotator.hs @@ -41,7 +41,7 @@ runLogsRotator TracerEnv } = do whenJust rotation \rotParams -> do traceWith teTracer TracerStartedLogRotator - launchRotator loggingParamsForFiles rotParams verbosity teRegistry teCurrentLogLock + launchRotator loggingParamsForFiles rotParams verbosity teTracer teRegistry teCurrentLogLock where loggingParamsForFiles :: [LoggingParams] loggingParamsForFiles = nub (NE.filter filesOnly logging) @@ -53,14 +53,15 @@ launchRotator :: [LoggingParams] -> RotationParams -> Maybe Verbosity + -> Trace IO TracerTrace -> HandleRegistry -> Lock -> IO () -launchRotator [] _ _ _ _ = return () +launchRotator [] _ _ _ _ _ = return () launchRotator loggingParamsForFiles - rotParams@RotationParams{rpFrequencySecs} verb registry currentLogLock = + rotParams@RotationParams{rpFrequencySecs} verb tracer registry currentLogLock = forever do - showProblemIfAny verb do + showProblemIfAny verb tracer do forM_ loggingParamsForFiles \loggingParam -> do checkRootDir currentLogLock registry rotParams loggingParam sleep $ fromIntegral rpFrequencySecs diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/TraceObjects.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/TraceObjects.hs index 097b6203361..dcf439b9c56 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/TraceObjects.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/TraceObjects.hs @@ -38,7 +38,7 @@ traceObjectsHandler _ _ _ [] = return () traceObjectsHandler tracerEnv _tracerEnvRTView nodeId traceObjects = do nodeName <- askNodeName tracerEnv nodeId forConcurrently_ logging \loggingParams@LoggingParams{logMode, logFormat} -> do - showProblemIfAny verbosity do + showProblemIfAny verbosity teTracer do case logMode of FileMode -> writeTraceObjectsToFile teRegistry @@ -56,6 +56,7 @@ traceObjectsHandler tracerEnv _tracerEnvRTView nodeId traceObjects = do , teCurrentLogLock , teReforwardTraceObjects , teRegistry + , teTracer } = tracerEnv deregisterNodeId :: TracerEnv -> NodeId -> IO () diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Metrics/Prometheus.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Metrics/Prometheus.hs index b60d5ad331c..68cf43b9646 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/Metrics/Prometheus.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Metrics/Prometheus.hs @@ -1,6 +1,7 @@ {-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE ViewPatterns #-} module Cardano.Tracer.Handlers.Metrics.Prometheus ( runPrometheusServer @@ -14,15 +15,19 @@ import Cardano.Tracer.MetaTrace import Prelude hiding (head) +import Control.Applicative ((<|>)) +import Data.Aeson (ToJSON (..), encode, pairs, (.=)) import qualified Data.ByteString as ByteString -import Data.ByteString.Builder (stringUtf8) import Data.Functor ((<&>)) -import Data.Text (Text) +import qualified Data.Map as Map (Map, empty, fromList) +import Data.Maybe +import Data.Text as T (Text, cons) +import qualified Data.Text.Encoding as T (decodeUtf8) import qualified Data.Text.Lazy as TL import Data.Text.Lazy.Builder (Builder) import qualified Data.Text.Lazy.Encoding as TL import Network.HTTP.Types -import Network.Wai hiding (responseHeaders) +import Network.Wai import Network.Wai.Handler.Warp (defaultSettings, runSettings) import System.Metrics as EKG (Store, sampleAll) import System.Time.Extra (sleep) @@ -30,7 +35,7 @@ import System.Time.Extra (sleep) -- | Runs a simple HTTP server that listens on @endpoint@. -- -- At the root, it lists the connected nodes, either as HTML or JSON, depending --- on the requests 'Accept: ' header. +-- on the request's 'Accept: ' header. -- -- Routing is dynamic, depending on the connected nodes. A valid URL is derived -- from the nodeName configured for the connecting node. E.g. a node name @@ -40,10 +45,6 @@ import System.Time.Extra (sleep) -- # TYPE Mem_resident_int gauge -- # HELP Mem_resident_int Kernel-reported RSS (resident set size) -- Mem_resident_int 103792640 --- # TYPE rts_gc_max_bytes_used gauge --- rts_gc_max_bytes_used 5811512 --- # TYPE rts_gc_gc_cpu_ms counter --- rts_gc_gc_cpu_ms 50 -- # TYPE RTS_gcMajorNum_int gauge -- # HELP RTS_gcMajorNum_int Major GCs -- RTS_gcMajorNum_int 4 @@ -56,6 +57,23 @@ import System.Time.Extra (sleep) -- # TYPE nodeCannotForge_int gauge -- # HELP nodeCannotForge_int How many times was this node unable to forge [a block]? -- +-- The `/targets` path can be used for Prometheus HTTP service discovery. This lets +-- Prometheus dynamically discover all connected nodes, and scrape their metrics. +-- Below is a minimal example of a corresponding job definition that goes into the +-- `prometheus.yml` configuration: +-- +-- - job_name: "cardano-tracer" +-- +-- http_sd_configs: +-- - url: 'http://127.0.0.1:3200/targets' # <-- Your cardano-tracer's real hostname:prometheus port +-- +-- Each target will have a label "node_name" which corresponds to the TraceOptionNodeName setting in the node config. +-- +-- In cardano-tracer's config, you can optionally provide additional labels to be attached to *all* targets +-- (default is no additional labels): +-- "prometheusLabels": { +-- "": "", ... +-- } runPrometheusServer :: TracerEnv -> Endpoint @@ -71,32 +89,27 @@ runPrometheusServer tracerEnv endpoint computeRoutes_autoUpdate = do { ttPrometheusEndpoint = endpoint } runSettings (setEndpoint endpoint defaultSettings) do - renderPrometheus computeRoutes_autoUpdate noSuffix teMetricsHelp + renderPrometheus computeRoutes_autoUpdate noSuffix teMetricsHelp promLabels where TracerEnv { teTracer - , teConfig = TracerConfig { metricsNoSuffix } + , teConfig = TracerConfig { metricsNoSuffix, prometheusLabels } , teMetricsHelp } = tracerEnv - noSuffix = or @Maybe metricsNoSuffix + noSuffix = or @Maybe metricsNoSuffix + promLabels = fromMaybe Map.empty prometheusLabels renderPrometheus :: IO RouteDictionary -> Bool -> [(Text, Builder)] + -> Map.Map Text Text -> Application -renderPrometheus computeRoutes_autoUpdate noSuffix helpTextDict request send = do +renderPrometheus computeRoutes_autoUpdate noSuffix helpTextDict promLabels request send = do routeDictionary :: RouteDictionary <- computeRoutes_autoUpdate - let acceptHeader :: Maybe ByteString.ByteString - acceptHeader = lookup hAccept $ requestHeaders request - - let wantsJson, wantsOpenMetrics :: Bool - wantsJson = all @Maybe ("application/json" `ByteString.isInfixOf`) acceptHeader - wantsOpenMetrics = all @Maybe ("application/openmetrics-text" `ByteString.isInfixOf`) acceptHeader - case pathInfo request of [] -> @@ -104,17 +117,51 @@ renderPrometheus computeRoutes_autoUpdate noSuffix helpTextDict request send = d then (contentHdrJSON , renderJson routeDictionary) else (contentHdrUtf8Html, renderListOfConnectedNodes "Prometheus metrics" routeDictionary) + ["targets"] + | wantsJson + -> serviceDiscovery routeDictionary + + | otherwise + -> wrongMType + route:_ | Just (store :: EKG.Store, _) <- lookup route (getRouteDictionary routeDictionary) - -> do metrics <- getMetricsFromNode noSuffix helpTextDict store - send $ responseBuilder status200 - (if wantsOpenMetrics then contentHdrOpenMetrics else contentHdrUtf8Text) - (TL.encodeUtf8Builder metrics) + -> metricsExposition store | otherwise - -> send $ responseBuilder status404 contentHdrUtf8Text do - "Not found: " - <> stringUtf8 (show route) + -> notFound route + + where + acceptHeader :: Maybe ByteString.ByteString + acceptHeader = lookup hAccept $ requestHeaders request + + wantsJson, wantsOpenMetrics :: Bool + wantsJson = all @Maybe ("application/json" `ByteString.isInfixOf`) acceptHeader + wantsOpenMetrics = all @Maybe ("application/openmetrics-text" `ByteString.isInfixOf`) acceptHeader + + -- we might support the more complex 'Forward:' header in the future + getHostNameRequest :: Maybe ByteString.ByteString + getHostNameRequest = + lookup "x-forwarded-host" (requestHeaders request) + <|> requestHeaderHost request + + metricsExposition store = do + metrics <- getMetricsFromNode noSuffix helpTextDict store + send $ responseBuilder status200 + (if wantsOpenMetrics then contentHdrOpenMetrics else contentHdrPrometheus) + (TL.encodeUtf8Builder metrics) + + serviceDiscovery (RouteDictionary routeDict) = + send $ responseLBS status200 contentHdrJSON $ + case getHostNameRequest of + Just (T.decodeUtf8 -> hostName) -> encode + [PSD (slug, nodeName, hostName, promLabels) | (slug, (_, nodeName)) <- routeDict] + Nothing -> "[]" + + notFound t = send $ responseLBS status404 contentHdrUtf8Text $ + "Not found: " <> (TL.encodeUtf8 . TL.fromStrict) t + wrongMType = send $ responseLBS status415 contentHdrUtf8Text + "Unsupported Media Type" getMetricsFromNode :: Bool @@ -123,3 +170,21 @@ getMetricsFromNode -> IO TL.Text getMetricsFromNode noSuffix helpTextDict ekgStore = sampleAll ekgStore <&> renderExpositionFromSampleWith helpTextDict noSuffix + + +-- This wrapper type implements the Prometheus HTTP SD format +-- cf. https://prometheus.io/docs/prometheus/latest/http_sd +-- It is local to this module, and never expected to provide an Aeson.Value. +newtype PrometheusServiceDiscovery = PSD (Text, Text, Text, Map.Map Text Text) + +instance ToJSON PrometheusServiceDiscovery where + toJSON _ = error "ToJSON.toJSON(PrometheusServiceDiscovery): implementation error" + + toEncoding (PSD (slug, nodeName, hostName, labelMap)) = pairs $ + ("targets" .= [hostName]) + <> ("labels" .= (labels <> labelMap)) + where + labels = Map.fromList + [ ("__metrics_path__", '/' `T.cons` slug) + , ("node_name" , nodeName) + ] diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Metrics/Utils.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Metrics/Utils.hs index a1cf8466065..438b5b392e8 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/Metrics/Utils.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Metrics/Utils.hs @@ -85,8 +85,9 @@ computeRoutes TracerEnv{teConnectedNodesNames, teAcceptedMetrics} = atomically d -contentHdrJSON, contentHdrOpenMetrics, contentHdrUtf8Html, contentHdrUtf8Text :: ResponseHeaders +contentHdrJSON, contentHdrOpenMetrics, contentHdrUtf8Html, contentHdrUtf8Text, contentHdrPrometheus :: ResponseHeaders contentHdrJSON = [(hContentType, "application/json")] -contentHdrOpenMetrics = [(hContentType, "application/openmetrics-text; version=1.0.0; charset=utf-8")] -contentHdrUtf8Html = [(hContentType, "text/html; charset=utf-8")] -contentHdrUtf8Text = [(hContentType, "text/plain; charset=utf-8")] +contentHdrOpenMetrics = [(hContentType, "application/openmetrics-text;version=1.0.0;charset=utf-8")] +contentHdrUtf8Html = [(hContentType, "text/html;charset=utf-8")] +contentHdrUtf8Text = [(hContentType, "text/plain;charset=utf-8")] +contentHdrPrometheus = [(hContentType, "text/plain;version=0.0.4;charset=utf-8")] diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/ReForwarder.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/ReForwarder.hs index 9fa619e45ea..b94e072ab29 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/ReForwarder.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/ReForwarder.hs @@ -1,7 +1,9 @@ {-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RecordWildCards #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} +{-# LANGUAGE ViewPatterns #-} -- | This module initializes a reforwarding service for use by -- cardano-tracer. It could [re-] serve the three miniprotocols on @@ -18,14 +20,14 @@ module Cardano.Tracer.Handlers.ReForwarder import Cardano.Logging.Trace import Cardano.Logging.Tracer.DataPoint import qualified Cardano.Logging.Types as Log -import qualified Cardano.Logging.Types as Net import Cardano.Tracer.Configuration +import Cardano.Tracer.Handlers.Utils (normalizeNamespace) import Cardano.Tracer.MetaTrace import Ouroboros.Network.Magic (NetworkMagic (..)) import Ouroboros.Network.NodeToClient (withIOManager) +import Control.Exception (SomeException (..)) import Control.Monad (when) -import Data.List (isPrefixOf) import qualified Data.Text as Text import Trace.Forward.Forwarding @@ -46,34 +48,17 @@ initReForwarder TracerConfig{networkMagic, hasForwarding} teTracer = do mForwarding <- case hasForwarding of Nothing -> pure Nothing - Just x -> case x of - (ConnectTo{}, _, _) -> + Just (ConnectTo{}, _, _) -> error "initReForwarder: unsupported mode of operation: ConnectTo. Use AcceptAt." - (AcceptAt (LocalPipe socket), mFwdNames, forwConf) -> do + Just (AcceptAt howToConnect, flattenNS -> mFwdNames, forwConf) -> do (fwdsink, dpStore :: DataPointStore) <- withIOManager \iomgr -> do traceWith teTracer TracerStartedReforwarder - initForwarding iomgr forwConf - (NetworkMagic networkMagic) - Nothing - (Just (Net.LocalPipe socket, Log.Responder)) + initForwarding iomgr forwConf $ initForwardingWith howToConnect pure $ Just ( filteredWriteToSink (traceObjectHasPrefixIn mFwdNames) fwdsink , dataPointTracer @IO dpStore ) - (AcceptAt (RemoteSocket host port), mFwdNames, forwConf) -> do - (fwdsink, dpStore :: DataPointStore) <- withIOManager \iomgr -> do - traceWith teTracer TracerStartedReforwarder - initForwarding iomgr forwConf - (NetworkMagic networkMagic) - Nothing - (Just (Net.RemoteSocket host port, Log.Responder)) - pure $ Just ( filteredWriteToSink - (traceObjectHasPrefixIn mFwdNames) - fwdsink - , dataPointTracer @IO dpStore - ) - let traceDP = case mForwarding of Just (_,tr) -> tr Nothing -> mempty @@ -86,14 +71,25 @@ initReForwarder TracerConfig{networkMagic, hasForwarding} const $ return () return (writesToSink', traceDP) + where + flattenNS = fmap (map (Text.intercalate ".")) + initForwardingWith initHowToConnect = + InitForwardingWith + { initNetworkMagic = NetworkMagic networkMagic + , initEKGStore = Nothing + , initForwarderMode = Log.Responder + , initOnForwardInterruption = Just $ \(SomeException e) -> + traceWith teTracer (TracerForwardingInterrupted initHowToConnect $ show e) + , initOnQueueOverflow = Nothing + , .. + } -traceObjectHasPrefixIn :: Maybe [[Text.Text]] -> Log.TraceObject -> Bool -traceObjectHasPrefixIn mFwdNames logObj = +traceObjectHasPrefixIn :: Maybe [Text.Text] -> Log.TraceObject -> Bool +traceObjectHasPrefixIn mFwdNames (normalizeNamespace . Log.toNamespace -> ns) = case mFwdNames of Nothing -> True -- forward everything in this case - Just fwdNames -> any (`isPrefixOf` Log.toNamespace logObj) fwdNames - + Just fwdNames -> any (`Text.isPrefixOf` ns) fwdNames filteredWriteToSink :: (Log.TraceObject -> Bool) -> ForwardSink Log.TraceObject diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/State/TraceObjects.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/State/TraceObjects.hs index aad84882262..f77a47ef8b0 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/State/TraceObjects.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/State/TraceObjects.hs @@ -1,5 +1,4 @@ {-# LANGUAGE NamedFieldPuns #-} -{-# LANGUAGE OverloadedStrings #-} module Cardano.Tracer.Handlers.State.TraceObjects ( LogsLiveViewCounters @@ -15,6 +14,7 @@ module Cardano.Tracer.Handlers.State.TraceObjects ) where import Cardano.Logging (SeverityS, TraceObject (..)) +import Cardano.Tracer.Handlers.Utils (normalizeNamespace) import Cardano.Tracer.Types (NodeId) import Control.Concurrent.STM (atomically) @@ -24,7 +24,7 @@ import Control.Monad (forM_, unless) import Data.Map.Strict (Map) import qualified Data.Map.Strict as M import Data.Maybe (mapMaybe) -import Data.Text (Text, intercalate) +import Data.Text as T (Text, null) import Data.Time.Clock (UTCTime) type Namespace = Text @@ -42,7 +42,7 @@ saveTraceObjects -> [TraceObject] -> IO () saveTraceObjects savedTraceObjects nodeId traceObjects = - unless (null itemsToSave) $ atomically $ do + unless (Prelude.null itemsToSave) $ atomically $ do savedTO' <- readTVar savedTraceObjects case M.lookup nodeId savedTO' of Nothing -> do @@ -61,13 +61,11 @@ saveTraceObjects savedTraceObjects nodeId traceObjects = itemsToSave = mapMaybe getTOValue traceObjects getTOValue :: TraceObject -> Maybe (Namespace, TraceObjectInfo) - getTOValue TraceObject{toNamespace, toHuman, toMachine, toSeverity, toTimestamp} = - case (toNamespace, toHuman, toMachine) of - ([], _, _) -> Nothing - (ns, _, msg) -> Just (mkName ns, (msg, toSeverity, toTimestamp)) - - mkName :: [Text] -> Namespace - mkName = intercalate "." + getTOValue TraceObject{toNamespace, toMachine, toSeverity, toTimestamp} = + let ns = normalizeNamespace toNamespace + in if T.null ns + then Nothing + else Just (ns, (toMachine, toSeverity, toTimestamp)) pushItemsToQueue = forM_ itemsToSave . writeTQueue diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Utils.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Utils.hs index 6e9cdf5ad45..4b7073ed38a 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/Utils.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Utils.hs @@ -1,14 +1,10 @@ {-# LANGUAGE LambdaCase #-} {-# LANGUAGE NumericUnderscores #-} +{-# LANGUAGE OverloadedStrings #-} module Cardano.Tracer.Handlers.Utils - ( askDataPoint - , utc2ns - , utc2s - , s2utc - , readInt - , nullTime - ) where + ( module Cardano.Tracer.Handlers.Utils) + where import Cardano.Tracer.Types @@ -16,7 +12,7 @@ import Control.Concurrent.Extra (Lock, withLock) import Control.Concurrent.STM.TVar (readTVarIO) import Data.Aeson (FromJSON, decode') import qualified Data.Map.Strict as M -import Data.Text (Text) +import Data.Text as T (Text, empty, intercalate, null) import Data.Text.Read (decimal) import Data.Time.Calendar (Day (..)) import Data.Time.Clock (UTCTime (..)) @@ -69,3 +65,10 @@ readInt t defInt = nullTime :: UTCTime nullTime = UTCTime (ModifiedJulianDay 0) 0 + +-- this handles both namespace values that come ["as.a.single.text"] and ["as", "individual", "segments"] +normalizeNamespace :: [Text] -> Text +normalizeNamespace = \case + [ns] | not (T.null ns) -> ns + ns@(_:_) -> T.intercalate "." ns + _ -> T.empty diff --git a/cardano-tracer/src/Cardano/Tracer/MetaTrace.hs b/cardano-tracer/src/Cardano/Tracer/MetaTrace.hs index 82e25a09da1..ee01fddc8a0 100644 --- a/cardano-tracer/src/Cardano/Tracer/MetaTrace.hs +++ b/cardano-tracer/src/Cardano/Tracer/MetaTrace.hs @@ -2,7 +2,6 @@ {-# LANGUAGE AllowAmbiguousTypes #-} {-# LANGUAGE DataKinds #-} -{-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE GADTs #-} @@ -28,23 +27,26 @@ import Cardano.Tracer.Types (NodeId (..), NodeName) import Data.Aeson hiding (Error) import qualified Data.Aeson as AE import qualified Data.Map.Strict as Map -import Data.Text (Text) -import qualified Data.Text as T -import GHC.Generics +import Data.Text as T (Text, pack) import qualified System.IO as Sys + + +rtViewConfigWarning :: Text +rtViewConfigWarning = "RTView requested in config but cardano-tracer was built without it" + data TracerTrace -- | Static information about the build. = TracerBuildInfo - { ttBuiltWithRTView :: !Bool + { ttBuiltWithRTView :: Bool } | TracerParamsAre - { ttConfigPath :: !FilePath - , ttStateDir :: !(Maybe FilePath) - , ttMinLogSeverity :: !(Maybe SeverityS) } + { ttConfigPath :: FilePath + , ttStateDir :: Maybe FilePath + , ttMinLogSeverity :: Maybe SeverityS } | TracerConfigIs - { ttConfig :: !TracerConfig - , ttWarnRTViewMissing :: !Bool + { ttConfig :: TracerConfig + , ttWarnRTViewMissing :: Bool } | TracerInitStarted | TracerInitEventQueues @@ -54,152 +56,135 @@ data TracerTrace } | TracerStartedLogRotator | TracerStartedPrometheus - { ttPrometheusEndpoint :: !Endpoint + { ttPrometheusEndpoint :: Endpoint } | TracerStartedMonitoring - { ttMonitoringEndpoint :: !Endpoint - , ttMonitoringType :: !Text + { ttMonitoringEndpoint :: Endpoint + , ttMonitoringType :: Text } | TracerStartedAcceptors - { ttAcceptorsAddr :: !Network } + { ttAcceptorsAddr :: Network } | TracerStartedRTView | TracerStartedReforwarder | TracerSockListen - { ttListenAt :: !FilePath } + { ttListenAt :: FilePath } | TracerSockIncoming - { ttConnectionIncomingAt :: !FilePath - , ttAddr :: !Text } + { ttConnectionIncomingAt :: FilePath + , ttAddr :: Text } | TracerSockConnecting - { ttConnectingTo :: !FilePath } + { ttConnectingTo :: FilePath } | TracerSockConnected - { ttConnectedTo :: !FilePath } + { ttConnectedTo :: FilePath } | TracerShutdownInitiated | TracerShutdownHistBackup | TracerShutdownComplete | TracerError - { ttError :: !Text } + { ttError :: Text } | TracerResource - { ttResource :: !ResourceStats } - deriving (Generic, Show) + { ttResource :: ResourceStats } + | TracerForwardingInterrupted + { ttConnection :: HowToConnect + , ttMessage :: String + } + deriving Show + + +instance LogFormatting TracerTrace where + forHuman t@TracerConfigIs{ttWarnRTViewMissing = True} = + rtViewConfigWarning <> ": " <> forHuman t {ttWarnRTViewMissing = False} + forHuman (TracerForwardingInterrupted howToConnect msg) = + T.pack $ "connection with " <> show howToConnect <> " failed: " <> msg + forHuman _ = "" -instance ToJSON TracerTrace where - toEncoding :: TracerTrace -> Encoding - toEncoding = \case - TracerBuildInfo{..} -> concatPairs - [ "BuiltWithRTView" .= ttBuiltWithRTView - , "kind" .= txt "TracerBuildInfo" + forMachine _dtal = \case + TracerBuildInfo{..} -> mconcat + [ "builtWithRTView" .= ttBuiltWithRTView + , "kind" .= AE.String "TracerBuildInfo" ] - TracerParamsAre{..} -> concatPairs - [ "ConfigPath" .= ttConfigPath - , "StateDir" .= ttStateDir - , "MinLogSeverity" .= ttMinLogSeverity - , "kind" .= txt "TracerParamsAre" + TracerParamsAre{..} -> mconcat + [ "configPath" .= ttConfigPath + , "stateDir" .= ttStateDir + , "minLogSeverity" .= ttMinLogSeverity + , "kind" .= AE.String "TracerParamsAre" ] - TracerConfigIs{..} -> concatPairs $ - [ "Config" .= ttConfig - , "kind" .= txt "TracerConfigIs" ] ++ - [ "WarnRTViewMissing" .= txt "RTView requested in config but cardano-tracer was built without it." + TracerConfigIs{..} -> mconcat $ + [ "config" .= ttConfig + , "kind" .= AE.String "TracerConfigIs" ] ++ + [ "warnRTViewMissing" .= rtViewConfigWarning | ttWarnRTViewMissing ] - TracerInitStarted -> concatPairs - [ "kind" .= txt "TracerInitStarted" + TracerInitStarted -> mconcat + [ "kind" .= AE.String "TracerInitStarted" ] - TracerInitEventQueues -> concatPairs - [ "kind" .= txt "TracerInitEventQueues" + TracerInitEventQueues -> mconcat + [ "kind" .= AE.String "TracerInitEventQueues" ] - TracerInitDone -> concatPairs - [ "kind" .= txt "TracerInitDone" + TracerInitDone -> mconcat + [ "kind" .= AE.String "TracerInitDone" ] - TracerAddNewNodeIdMapping (NodeId nodeId, nodeName) -> concatPairs - [ "kind" .= txt "TracerAddNewNodeIdMapping" - , "nodeId" .= txt nodeId - , "nodeName" .= txt nodeName + TracerAddNewNodeIdMapping (NodeId nodeId, nodeName) -> mconcat + [ "kind" .= AE.String "TracerAddNewNodeIdMapping" + , "nodeId" .= AE.String nodeId + , "nodeName" .= AE.String nodeName ] - TracerStartedLogRotator -> concatPairs - [ "kind" .= txt "TracerStartedLogRotator" + TracerStartedLogRotator -> mconcat + [ "kind" .= AE.String "TracerStartedLogRotator" ] - TracerStartedPrometheus{..} -> concatPairs - [ "kind" .= txt "TracerStartedPrometheus" + TracerStartedPrometheus{..} -> mconcat + [ "kind" .= AE.String "TracerStartedPrometheus" , "endpoint" .= ttPrometheusEndpoint ] - TracerStartedMonitoring{..} -> concatPairs - [ "kind" .= txt "TracerStartedMonitoring" + TracerStartedMonitoring{..} -> mconcat + [ "kind" .= AE.String "TracerStartedMonitoring" , "endpoint" .= ttMonitoringEndpoint , "type" .= ttMonitoringType ] - TracerStartedAcceptors{..} -> concatPairs - [ "kind" .= txt "TracerStartedAcceptors" + TracerStartedAcceptors{..} -> mconcat + [ "kind" .= AE.String "TracerStartedAcceptors" , "AcceptorsAddr" .= ttAcceptorsAddr ] - TracerStartedRTView -> concatPairs - [ "kind" .= txt "TracerStartedRTView" + TracerStartedRTView -> mconcat + [ "kind" .= AE.String "TracerStartedRTView" ] - TracerStartedReforwarder -> concatPairs - [ "kind" .= txt "TracerStartedReforwarder" + TracerStartedReforwarder -> mconcat + [ "kind" .= AE.String "TracerStartedReforwarder" ] - TracerSockListen{..} -> concatPairs - [ "kind" .= txt "TracerSockListen" - , "ListenAt" .= ttListenAt + TracerSockListen{..} -> mconcat + [ "kind" .= AE.String "TracerSockListen" + , "listenAt" .= ttListenAt ] - TracerSockIncoming{..} -> concatPairs - [ "kind" .= txt "TracerSockIncoming" - , "ConnectionIncomingAt" .= ttConnectionIncomingAt - , "Addr" .= ttAddr + TracerSockIncoming{..} -> mconcat + [ "kind" .= AE.String "TracerSockIncoming" + , "connectionIncomingAt" .= ttConnectionIncomingAt + , "addr" .= ttAddr ] - TracerSockConnecting{..} -> concatPairs - [ "kind" .= txt "TracerSockConnecting" - , "ConnectionIncomingAt" .= ttConnectingTo + TracerSockConnecting{..} -> mconcat + [ "kind" .= AE.String "TracerSockConnecting" + , "connectionIncomingAt" .= ttConnectingTo ] - TracerSockConnected{..} -> concatPairs - [ "kind" .= txt "TracerSockConnected" - , "ConnectedTo" .= ttConnectedTo + TracerSockConnected{..} -> mconcat + [ "kind" .= AE.String "TracerSockConnected" + , "connectedTo" .= ttConnectedTo ] - TracerShutdownInitiated -> concatPairs - [ "kind" .= txt "TracerShutdownInitiated" + TracerShutdownInitiated -> mconcat + [ "kind" .= AE.String "TracerShutdownInitiated" ] - TracerShutdownHistBackup -> concatPairs - [ "kind" .= txt "TracerShutdownHistBackup" + TracerShutdownHistBackup -> mconcat + [ "kind" .= AE.String "TracerShutdownHistBackup" ] - TracerShutdownComplete -> concatPairs - [ "kind" .= txt "TracerShutdownComplete" + TracerShutdownComplete -> mconcat + [ "kind" .= AE.String "TracerShutdownComplete" ] - TracerError{..} -> concatPairs - [ "kind" .= txt "TracerError" - , "Error" .= ttError + TracerError{..} -> mconcat + [ "kind" .= AE.String "TracerError" + , "error" .= ttError ] - TracerResource{..} -> concatPairs - [ "kind" .= txt "TracerResource" - , "Resource" .= ttResource - ] - where - txt :: Text -> Text - txt = id - concatPairs :: [Series] -> Encoding - concatPairs = pairs . mconcat - - toJSON = AE.genericToJSON jsonEncodingOptions - -jsonEncodingOptions :: AE.Options -jsonEncodingOptions = AE.defaultOptions - { AE.fieldLabelModifier = drop 2 - , AE.tagSingleConstructors = True - , AE.sumEncoding = - AE.TaggedObject - { AE.tagFieldName = "kind" - , AE.contentsFieldName = "contents" - } - } - -instance LogFormatting TracerTrace where - forHuman t@TracerConfigIs{ttWarnRTViewMissing = True} = T.pack $ - unlines - [ show t ++ ": RTView requested in config but cardano-tracer was built without it." - , "Enable with `-f +rtview`." + TracerResource{..} -> forMachine _dtal ttResource + TracerForwardingInterrupted{..} -> mconcat + [ "kind" .= AE.String "TracerForwardingInterrupted" + , "conn" .= ttConnection + , "message" .= ttMessage ] - forHuman t = T.pack (show t) - forMachine _ t = case AE.toJSON t of - AE.Object x -> x - _ -> error "Impossible" instance MetaTrace TracerTrace where namespaceFor TracerBuildInfo {} = Namespace [] ["BuildInfo"] @@ -223,7 +208,8 @@ instance MetaTrace TracerTrace where namespaceFor TracerShutdownHistBackup = Namespace [] ["ShutdownHistBackup"] namespaceFor TracerShutdownComplete = Namespace [] ["ShutdownComplete"] namespaceFor TracerError {} = Namespace [] ["Error"] - namespaceFor TracerResource {} = Namespace [] ["Resource"] + namespaceFor TracerResource {} = Namespace [] ["Resources"] + namespaceFor TracerForwardingInterrupted {} = Namespace [] ["ForwardingInterrupted"] severityFor (Namespace _ ["BuildInfo"]) _ = Just Info severityFor (Namespace _ ["ParamsAre"]) _ = Just Warning @@ -246,7 +232,8 @@ instance MetaTrace TracerTrace where severityFor (Namespace _ ["ShutdownHistBackup"]) _ = Just Info severityFor (Namespace _ ["ShutdownComplete"]) _ = Just Warning severityFor (Namespace _ ["Error"]) _ = Just Error - severityFor (Namespace _ ["Resource"]) _ = Just Info + severityFor (Namespace _ ["Resources"]) _ = Just Info + severityFor (Namespace _ ["ForwardingInterrupted"]) _ = Just Warning severityFor _ _ = Nothing documentFor _ = Just "" @@ -273,7 +260,8 @@ instance MetaTrace TracerTrace where , Namespace [] ["ShutdownHistBackup"] , Namespace [] ["ShutdownComplete"] , Namespace [] ["Error"] - , Namespace [] ["Resource"] + , Namespace [] ["Resources"] + , Namespace [] ["ForwardingInterrupted"] ] stderrShowTracer :: Trace IO TracerTrace diff --git a/cardano-tracer/src/Cardano/Tracer/Utils.hs b/cardano-tracer/src/Cardano/Tracer/Utils.hs index b87ddb1efb5..4d802314125 100644 --- a/cardano-tracer/src/Cardano/Tracer/Utils.hs +++ b/cardano-tracer/src/Cardano/Tracer/Utils.hs @@ -2,7 +2,6 @@ {-# LANGUAGE LambdaCase #-} {-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE PackageImports #-} {-# LANGUAGE ScopedTypeVariables #-} #if !defined(mingw32_HOST_OS) @@ -21,11 +20,9 @@ module Cardano.Tracer.Utils , initConnectedNodesNames , initDataPointRequestors , initProtocolsBrake - , logTrace , forMM , forMM_ , nl - , runInLoop , showProblemIfAny , memberRegistry , showRegistry @@ -40,11 +37,11 @@ module Cardano.Tracer.Utils ) where import Cardano.Logging.Types.NodeInfo (NodeInfo(..)) +import Cardano.Logging.Utils (showT) import Cardano.Tracer.Configuration import Cardano.Tracer.Environment import Cardano.Tracer.Handlers.Utils -import qualified Cardano.Logging as Tracer (traceWith) -import Cardano.Tracer.MetaTrace hiding (traceWith) +import Cardano.Tracer.MetaTrace import Cardano.Tracer.Types import Ouroboros.Network.Socket (ConnectionId (..)) @@ -54,11 +51,9 @@ import Control.Concurrent.Extra (Lock) import Control.Concurrent.MVar (newMVar, swapMVar, readMVar, tryReadMVar, modifyMVar_) import Control.Concurrent.STM (atomically) import Control.Concurrent.STM.TVar (modifyTVar', stateTVar, readTVarIO, newTVarIO) -import Control.Exception (SomeAsyncException (..), SomeException, finally, - fromException, try, tryJust, throwTo) +import Control.Exception (SomeException, finally, throwTo, try) import Control.Monad (forM_) import Control.Monad.Extra (whenJustM) -import "contra-tracer" Control.Tracer (stdoutTracer, traceWith) import Data.Word (Word32) import qualified Data.Bimap as BM import Data.Bimap (Bimap) @@ -72,7 +67,6 @@ import System.Exit (ExitCode (ExitSuccess)) import System.IO (hClose, hFlush, stdout) import System.Mem.Weak (deRefWeak) import qualified System.Signal as Signal -import System.Time.Extra (sleep) #if defined(mingw32_HOST_OS) import System.Win32.Process (getCurrentProcessId) @@ -81,52 +75,20 @@ import System.Posix.Process (getProcessID) import System.Posix.Types (CPid (..)) #endif --- | Run monadic action in a loop. If there's an exception, --- it will re-run the action again, after pause that grows. -runInLoop - :: IO () -- ^ An IO-action that can throw an exception. - -> Maybe Verbosity -- ^ Tracer's verbosity. - -> HowToConnect -- ^ Means of connecting, by local or remote socket. - -> Word -- ^ Current delay, in seconds. - -> IO () -runInLoop action verbosity howToConnect prevDelay = - tryJust excludeAsyncExceptions action >>= \case - Left err -> do - case verbosity of - Just Minimum -> return () - _ -> logTrace $ "cardano-tracer, connection with " <> show howToConnect <> " failed: " <> show @SomeException err - sleep $ fromIntegral currentDelay - runInLoop action verbosity howToConnect currentDelay - Right {} -> - pure () - where - excludeAsyncExceptions :: SomeException -> Maybe SomeException - excludeAsyncExceptions e = - case fromException e of - Just SomeAsyncException {} -> Nothing - _ -> Just e - - currentDelay :: Word - !currentDelay = - if prevDelay < 60 - then prevDelay * 2 - else 60 -- After we reached 60+ secs delay, repeat an attempt every minute. showProblemIfAny - :: Maybe Verbosity -- ^ Tracer's verbosity. - -> IO () -- ^ An IO-action that can throw an exception. + :: Maybe Verbosity -- ^ Tracer's verbosity. + -> Trace IO TracerTrace -- ^ Trace an error with that tracer iff not at minimum verbosity + -> IO () -- ^ An IO action that can throw an exception. -> IO () -showProblemIfAny verb action = +showProblemIfAny verb tracer action = try action >>= \case Left (e :: SomeException) -> case verb of Just Minimum -> return () - _ -> logTrace $ "cardano-tracer, the problem: " <> show e + _ -> traceWith tracer $ TracerError $ showT e Right _ -> return () -logTrace :: String -> IO () -logTrace = traceWith stdoutTracer - connIdToNodeId :: Show addr => ConnectionId addr -> NodeId connIdToNodeId ConnectionId{remoteAddress} = NodeId preparedAddress where @@ -207,7 +169,7 @@ askNodeNameRaw tracer connectedNodesNames dpRequestors currentDPLock nodeId@(Nod in (maybePair, newBimap) for_ @Maybe maybePair \pair -> - Tracer.traceWith tracer TracerAddNewNodeIdMapping + traceWith tracer TracerAddNewNodeIdMapping { ttBimapping = pair } diff --git a/cardano-tracer/test/Cardano/Tracer/Test/Acceptor.hs b/cardano-tracer/test/Cardano/Tracer/Test/Acceptor.hs index e998e2f6811..e26ec077675 100644 --- a/cardano-tracer/test/Cardano/Tracer/Test/Acceptor.hs +++ b/cardano-tracer/test/Cardano/Tracer/Test/Acceptor.hs @@ -119,6 +119,7 @@ launchAcceptorsSimple mode localSock dpName = do , hasForwarding = Nothing , resourceFreq = Nothing , ekgRequestFull = Nothing + , prometheusLabels = Nothing } -- | To be able to ask any 'DataPoint' by the name without knowing the actual type, diff --git a/cardano-tracer/test/Cardano/Tracer/Test/DataPoint/Tests.hs b/cardano-tracer/test/Cardano/Tracer/Test/DataPoint/Tests.hs index 39d9fc2177c..49ba0cf4183 100644 --- a/cardano-tracer/test/Cardano/Tracer/Test/DataPoint/Tests.hs +++ b/cardano-tracer/test/Cardano/Tracer/Test/DataPoint/Tests.hs @@ -96,4 +96,5 @@ propDataPoint ts@TestSetup{..} rootDir localSock = do , hasForwarding = Nothing , resourceFreq = Nothing , ekgRequestFull = Nothing + , prometheusLabels = Nothing } diff --git a/cardano-tracer/test/Cardano/Tracer/Test/Forwarder.hs b/cardano-tracer/test/Cardano/Tracer/Test/Forwarder.hs index 841a26dc14e..de95bef2a5d 100644 --- a/cardano-tracer/test/Cardano/Tracer/Test/Forwarder.hs +++ b/cardano-tracer/test/Cardano/Tracer/Test/Forwarder.hs @@ -18,6 +18,7 @@ module Cardano.Tracer.Test.Forwarder import Cardano.Logging (DetailLevel (..), SeverityS (..), TraceObject (..)) import Cardano.Logging.Types (HowToConnect) import qualified Cardano.Logging.Types as Net +import Cardano.Logging.Utils (runInLoop) import Cardano.Tracer.Configuration (Verbosity (..)) import Cardano.Tracer.Test.TestSetup import Cardano.Tracer.Test.Utils @@ -43,7 +44,8 @@ import Control.Concurrent.Async hiding (async) import Control.DeepSeq (NFData) import Control.Exception (IOException, SomeException, catch, throwIO, try) import Control.Monad (forever) -import "contra-tracer" Control.Tracer (contramap, nullTracer, stdoutTracer) +import "contra-tracer" Control.Tracer as Contra (contramap, nullTracer, stdoutTracer, + traceWith) import Data.Aeson (FromJSON, ToJSON) import qualified Data.ByteString.Lazy as LBS import Data.Functor (void) @@ -93,7 +95,9 @@ launchForwardersSimple -> Word -> IO () launchForwardersSimple ts mode howToConnect queueSize = withIOManager \iomgr -> - runInLoop (launchForwardersSimple' ts iomgr mode howToConnect queueSize) (Just Minimum) howToConnect 1 + runInLoop (launchForwardersSimple' ts iomgr mode howToConnect queueSize) handleInterruption 1 60 + where + handleInterruption = const $ pure () launchForwardersSimple' :: TestSetup Identity @@ -321,3 +325,6 @@ traceObjectsWriter sink = forever do , toHostname = "nixos" , toThreadId = "1" } + +logTrace :: String -> IO () +logTrace = Contra.traceWith Contra.stdoutTracer diff --git a/cardano-tracer/test/Cardano/Tracer/Test/Logs/Tests.hs b/cardano-tracer/test/Cardano/Tracer/Test/Logs/Tests.hs index 77d3e9eae52..746d3a5e8d6 100644 --- a/cardano-tracer/test/Cardano/Tracer/Test/Logs/Tests.hs +++ b/cardano-tracer/test/Cardano/Tracer/Test/Logs/Tests.hs @@ -82,6 +82,7 @@ propLogs ts@TestSetup{..} format logRotLimitBytes logRotMaxAgeMinutes rootDir lo , hasForwarding = Nothing , resourceFreq = Nothing , ekgRequestFull = Nothing + , prometheusLabels = Nothing } propMultiInit :: TestSetup Identity -> LogFormat -> FilePath -> HowToConnect -> HowToConnect -> IO Property @@ -121,6 +122,7 @@ propMultiInit ts@TestSetup{..} format rootDir howToConnect1 howToConnect2 = do , hasForwarding = Nothing , resourceFreq = Nothing , ekgRequestFull = Nothing + , prometheusLabels = Nothing } -- | Tests @@ -163,6 +165,7 @@ propMultiResp ts@TestSetup{..} format rootDir howToConnect = do , hasForwarding = Nothing , resourceFreq = Nothing , ekgRequestFull = Nothing + , prometheusLabels = Nothing } checkPropLogsResults :: FilePath -> LogFormat -> IO Property diff --git a/cardano-tracer/test/Cardano/Tracer/Test/Restart/Tests.hs b/cardano-tracer/test/Cardano/Tracer/Test/Restart/Tests.hs index afebe154f60..a1441badbf5 100644 --- a/cardano-tracer/test/Cardano/Tracer/Test/Restart/Tests.hs +++ b/cardano-tracer/test/Cardano/Tracer/Test/Restart/Tests.hs @@ -102,4 +102,5 @@ mkConfig TestSetup{..} rootDir p = TracerConfig , hasForwarding = Nothing , resourceFreq = Nothing , ekgRequestFull = Nothing + , prometheusLabels = Nothing } diff --git a/cardano-tracer/test/cardano-tracer-test-ext.hs b/cardano-tracer/test/cardano-tracer-test-ext.hs index d8ef8fcbac5..92a156b4f4e 100644 --- a/cardano-tracer/test/cardano-tracer-test-ext.hs +++ b/cardano-tracer/test/cardano-tracer-test-ext.hs @@ -30,7 +30,7 @@ import qualified System.Process as Sys import Test.Tasty import Test.Tasty.QuickCheck -import Trace.Forward.Forwarding (initForwarding) +import Trace.Forward.Forwarding (InitForwardingConfig (..), initForwarding) import Trace.Forward.Utils.TraceObject (writeToSink) main :: IO () @@ -137,7 +137,14 @@ getExternalTracerState TestSetup{..} ref = do (forwardSink, _dpStore) <- withIOManager \iomgr -> do -- For simplicity, we are always 'Initiator', -- so 'cardano-tracer' is always a 'Responder'. - let tracerSocketMode = Just (Net.LocalPipe (unI tsSockExternal), Initiator) - forwardingConf = fromMaybe defaultForwarder (tcForwarder simpleTestConfig) - initForwarding iomgr forwardingConf (unI tsNetworkMagic) Nothing tracerSocketMode + let forwardingConf = fromMaybe defaultForwarder (tcForwarder simpleTestConfig) + initForwarding iomgr forwardingConf $ + InitForwardingWith + { initNetworkMagic = unI tsNetworkMagic + , initEKGStore = Nothing + , initHowToConnect = Net.LocalPipe (unI tsSockExternal) + , initForwarderMode = Initiator + , initOnForwardInterruption = Nothing + , initOnQueueOverflow = Nothing + } pure (externalTracerHdl, forwardTracer (writeToSink forwardSink)) diff --git a/trace-dispatcher/CHANGELOG.md b/trace-dispatcher/CHANGELOG.md index 2c2ac5fb390..9b9e429a928 100644 --- a/trace-dispatcher/CHANGELOG.md +++ b/trace-dispatcher/CHANGELOG.md @@ -1,8 +1,14 @@ # Revision history for trace-dispatcher -## NEXT -- Nov 2025 - -* Replaced both `tofDisconnQueueSize` and `tofConnQueueSize` with `tofQueueSize` (See #6361 for details). +## 2.11.0 -- Nov 2025 + +* `class LogFormatting`: remove redundant `forHumanFromMachine` and `forHumanOrMachine` (the system already does that inherently) +* Introduce type `Cardano.Logging.Types.TraceMessage.TraceMessage` with explicit codecs for JSON and CBOR +* Rework `PreFormatted` type and formatters to use `TraceMessage`; slightly optimize `humanFormatter'` +* Add CBOR formatting via `FormattedMessage.FormattedCBOR` constructor and a `cborFormatter'` function +* Replaced both `disconnectedQueueSize` and `connectedQueueSize` with `queueSize` in `TraceOptionForwarder` while keeping config parsing backwards compatible +* Add retry delay reset in `runInLoop` when the action runs sufficiently long +* Safely stop `standardTracer`'s stdout thread when there are no more producers ## 2.10.0 -- July, 2025 * Forwarding protocol supports connections over TCP socket, in addition to Unix domain sockets. diff --git a/trace-dispatcher/doc/trace-dispatcher.md b/trace-dispatcher/doc/trace-dispatcher.md index 89ebb0c1f86..582c646f46f 100644 --- a/trace-dispatcher/doc/trace-dispatcher.md +++ b/trace-dispatcher/doc/trace-dispatcher.md @@ -186,11 +186,11 @@ For the effective integration of trace messages into the tracing system, two ess The `LogFormatting` typeclass governs the presentation of trace messages, encompassing the mapping of traces to metrics and messages. It includes the following methods: -- The `forMachine` method caters to a machine-readable representation, adaptable based on the detail level. Implementation is mandatory for the trace author. +- The `forMachine` method caters to a machine-readable representation, adaptable based on the detail level. Implementation is mandatory for the trace author. The system will render this, +along with trace metadata, as JSON of type `Cardano.Logging.Types.TraceMessage.TraceMessage`. - The `forHuman` method renders the message in a human-readable form. Its default implementation is an -empty text, which will be replaced by forMachine in the rendering, if forHuman is empty. The forMachine -will by default rendered with a DNormal detsil level, if no other information is provided. +empty text. Whenever the system encounters the empty text, it will replace it with the machine-readable JSON, rendering it as a value in `{"data": }`, preventing potential loss of log information - The `asMetrics` method portrays the message as 0 to n metrics. The default implementation assumes no metrics. Each metric can optionally specify a hierarchical identifier as a `[Text]`. @@ -700,14 +700,14 @@ logs and forwarded tracing output. As mentioned earlier, trace backends serve as the final destinations for all traces once they have undergone trace interpretation, resulting in metrics and messages. The system defines three trace backends: -1. __Standard Tracer:__ This is the fundamental standard output tracer. Notably, it can accept both regular and confidential traces. It's important to construct only one standard tracer in any application, as attempting to create a new one will result in an exception. +1. __Standard Tracer:__ This is the fundamental standard output tracer. Notably, it can accept both regular and confidential traces. ```haskell standardTracer :: forall m. (MonadIO m) => m (Trace m FormattedMessage) ``` -2. __Trace-Forward Tracer:__ This is a network-only sink dedicated to forwarding messages using typed protocols over TCP or local sockets. It exclusively handles public traces and should be instantiated only once per application. +2. __Trace-Forward Tracer:__ This is a network-only sink dedicated to forwarding messages using typed protocols over TCP or local sockets. It exclusively handles public traces. ```haskell forwardTracer :: forall m. (MonadIO m) @@ -715,7 +715,7 @@ As mentioned earlier, trace backends serve as the final destinations for all tra -> Trace m FormattedMessage ``` -3. __EKG Tracer:__ This tracer submits metrics to a local EKG store, which then further forwards the messages. +3. __EKG Tracer:__ This tracer submits metrics to a local EKG store (which then can be exposed directly via the `PrometheusSimple` backend and/or forwarded). ```haskell ekgTracer :: MonadIO m @@ -723,7 +723,7 @@ As mentioned earlier, trace backends serve as the final destinations for all tra -> m (Trace m FormattedMessage) ``` -It's imperative to note that constructing more than one instance of each tracer in an application may lead to exceptions and should be avoided. +It's imperative to note that constructing more than one instance of each tracer in an application should absolutely be avoided, as it may result in unexpected behaviour. ## Data Points Overview and Deprecation Notice diff --git a/trace-dispatcher/src/Cardano/Logging/Formatter.hs b/trace-dispatcher/src/Cardano/Logging/Formatter.hs index 16de345dd87..5292df95bdb 100644 --- a/trace-dispatcher/src/Cardano/Logging/Formatter.hs +++ b/trace-dispatcher/src/Cardano/Logging/Formatter.hs @@ -13,49 +13,41 @@ module Cardano.Logging.Formatter ( , forwardFormatter' , machineFormatter , machineFormatter' + , cborFormatter + , cborFormatter' , humanFormatter , humanFormatter' ) where import Cardano.Logging.Trace (contramapM) import Cardano.Logging.Types +import Cardano.Logging.Types.TraceMessage +import Codec.Serialise (serialise) import Control.Concurrent (myThreadId) import Control.Monad.IO.Class (MonadIO, liftIO) import qualified Control.Tracer as T import Data.Aeson ((.=)) import qualified Data.Aeson as AE import qualified Data.Aeson.Encoding as AE +import qualified Data.ByteString.Lazy as BL (toStrict) import Data.Functor.Contravariant import Data.Maybe (fromMaybe) import Data.Text as T (Text, intercalate, null, pack) import Data.Text.Lazy (toStrict) import Data.Text.Lazy.Builder as TB import Data.Text.Lazy.Encoding (decodeUtf8) -import Data.Time (UTCTime, defaultTimeLocale, formatTime, getCurrentTime) +import Data.Time (defaultTimeLocale, formatTime, getCurrentTime) import Network.HostName import System.IO.Unsafe (unsafePerformIO) -encodingToText :: AE.Encoding -> Text -{-# INLINE encodingToText #-} -encodingToText = toStrict . decodeUtf8 . AE.encodingToLazyByteString - -timeFormatted :: UTCTime -> Text -{-# INLINE timeFormatted #-} -timeFormatted = pack . formatTime defaultTimeLocale "%F %H:%M:%S%4QZ" - -- If the hostname in the logs should be anything different from the system reported hostname, -- a new field would need to be added to PreFormatted to carry a new hostname argument to preFormatted. hostname :: Text {-# NOINLINE hostname #-} hostname = unsafePerformIO $ T.pack <$> getHostName --- This allows data sharing of an Encoding value, avoiding reconstruction of the underlying Builder -instance AE.ToJSON AE.Encoding where - toJSON = error "ToJSON(Aeson.Encoding): must never be called" - toEncoding = id - -- | Format this trace as metrics metricsFormatter @@ -86,22 +78,19 @@ preFormatted withForHuman = (lc, Right msg) -> do time <- liftIO getCurrentTime threadId <- liftIO myThreadId - let ns' = lcNSPrefix lc ++ lcNSInner lc - threadTextShortened = T.pack $ drop 9 $ show threadId -- drop "ThreadId " prefix - details = fromMaybe DNormal (lcDetails lc) - condForHuman = let txt = forHuman msg in if T.null txt then Nothing else Just txt - machineFormatted = AE.toEncoding $ forMachine details msg - - pure (lc, Right (PreFormatted - { pfForHuman = if withForHuman then condForHuman else Nothing - , pfForMachine = machineFormatted - , pfTimestamp = timeFormatted time - , pfTime = time - , pfNamespace = ns' - , pfThreadId = threadTextShortened - })) + let + pf = PreFormatted + { pfTime = time + , pfNamespace = intercalate "." (lcNSPrefix lc ++ lcNSInner lc) + , pfThreadId = T.pack $ drop 9 $ show threadId -- drop "ThreadId " prefix + , pfForHuman = if withForHuman then (let txt = forHuman msg in if T.null txt then Nothing else Just txt) else Nothing + , pfForMachineObject = forMachine (fromMaybe DNormal (lcDetails lc)) msg + } + pure (lc, Right pf) + (lc, Left ctrl) -> - pure (lc, Left ctrl)) + pure (lc, Left ctrl) + ) -- | Format this trace as TraceObject for the trace forwarder forwardFormatter' @@ -113,28 +102,31 @@ forwardFormatter' (Trace tr) = Trace $ contramap (\ case (lc, Right v) -> - let machineObj = AE.pairs $ - "at" .= pfTime v - <> "ns" .= intercalate "." (pfNamespace v) - <> "data" .= pfForMachine v - <> "sev" .= fromMaybe Info (lcSeverity lc) - <> "thread" .= pfThreadId v - <> "host" .= hostname - to = TraceObject { - toHuman = pfForHuman v - , toMachine = encodingToText machineObj - , toNamespace = pfNamespace v - , toSeverity = fromMaybe Info (lcSeverity lc) - , toDetails = fromMaybe DNormal (lcDetails lc) - , toTimestamp = pfTime v - , toHostname = hostname - , toThreadId = pfThreadId v + let + jsonObj = TraceMessage + { tmsgAt = pfTime v + , tmsgNS = pfNamespace v + , tmsgData = pfForMachineObject v + , tmsgSev = fromMaybe Info $ lcSeverity lc + , tmsgThread = pfThreadId v + , tmsgHost = hostname + } + to = TraceObject + { toHuman = pfForHuman v + , toMachine = (toStrict . decodeUtf8 . AE.encode) jsonObj + -- backwards compatible to not break ForwardingV_1 protocol' type: value used to be segmented (["name", "space"]) + , toNamespace = [pfNamespace v] + , toSeverity = fromMaybe Info (lcSeverity lc) + , toDetails = fromMaybe DNormal (lcDetails lc) + , toTimestamp = pfTime v + , toHostname = hostname + , toThreadId = pfThreadId v } in (lc, Right (FormattedForwarder to)) (lc, Left ctrl) -> (lc, Left ctrl)) tr --- | Format this trace as TraceObject for the trace forwarder +-- | Format this trace as TraceObject for machine-readable text output (JSON) machineFormatter' :: forall m . MonadIO m @@ -144,18 +136,43 @@ machineFormatter' (Trace tr) = Trace $ contramap (\ case (lc, Right v) -> - let machineObj = AE.pairs $ - "at" .= pfTime v - <> "ns" .= intercalate "." (pfNamespace v) - <> "data" .= pfForMachine v - <> "sev" .= fromMaybe Info (lcSeverity lc) - <> "thread" .= pfThreadId v - <> "host" .= hostname - in (lc, Right (FormattedMachine (encodingToText machineObj))) + let + msg = TraceMessage + { tmsgAt = pfTime v + , tmsgNS = pfNamespace v + , tmsgData = pfForMachineObject v + , tmsgSev = fromMaybe Info $ lcSeverity lc + , tmsgThread = pfThreadId v + , tmsgHost = hostname + } + in (lc, Right (FormattedMachine (toStrict . decodeUtf8 $ AE.encode msg))) + (lc, Left ctrl) -> (lc, Left ctrl)) + tr + +-- | Format this trace in binary serialisation (CBOR) +cborFormatter' + :: forall m . + MonadIO m + => Trace m FormattedMessage + -> Trace m PreFormatted +cborFormatter' (Trace tr) = Trace $ + contramap + (\ case + (lc, Right v) -> + let + cborObj = TraceMessage + { tmsgAt = pfTime v + , tmsgNS = pfNamespace v + , tmsgData = pfForMachineObject v + , tmsgSev = fromMaybe Info $ lcSeverity lc + , tmsgThread = pfThreadId v + , tmsgHost = hostname + } + in (lc, Right (FormattedCBOR $ BL.toStrict $ serialise cborObj)) (lc, Left ctrl) -> (lc, Left ctrl)) tr --- | Format this trace in human readable style +-- | Format this trace in human readable text output humanFormatter' :: forall m . MonadIO m @@ -170,15 +187,18 @@ humanFormatter' withColor (Trace tr) = let sev = fromMaybe Info (lcSeverity lc) ns = fromText hostname <> singleton ':' - <> fromText (intercalate "." (pfNamespace v)) - prePart = squareBrackets (fromText (pfTimestamp v)) + <> fromText (pfNamespace v) + showTime = formatTime defaultTimeLocale "%F %H:%M:%S%4QZ" + prePart = squareBrackets (fromString $ showTime $ pfTime v) <> squareBrackets ns <> roundBrackets (fromString (show sev) <> singleton ',' <> fromText (pfThreadId v)) dataPart = fromMaybe - (encodingToText (AE.pairs ("data" .= pfForMachine v))) + (toStrict . decodeUtf8 . AE.encodingToLazyByteString $ + AE.pairs ("data" .= pfForMachineObject v) + ) (pfForHuman v) forHuman'' = toStrict $ toLazyText @@ -237,6 +257,15 @@ machineFormatter machineFormatter = preFormatted False . machineFormatter' +cborFormatter + :: forall a m . + (MonadIO m + , LogFormatting a) + => Trace m FormattedMessage + -> m (Trace m a) +cborFormatter = + preFormatted False . cborFormatter' + forwardFormatter :: forall a m . MonadIO m diff --git a/trace-dispatcher/src/Cardano/Logging/Prometheus/TCPServer.hs b/trace-dispatcher/src/Cardano/Logging/Prometheus/TCPServer.hs index 8073dcb0160..6f04948b28c 100644 --- a/trace-dispatcher/src/Cardano/Logging/Prometheus/TCPServer.hs +++ b/trace-dispatcher/src/Cardano/Logging/Prometheus/TCPServer.hs @@ -87,16 +87,14 @@ buildResponse getCurrentExposition = \case | method == UNSUPPORTED -> pure $ responseError withBody errorBadMethod | accept == Unsupported -> pure $ responseError withBody errorBadContent | otherwise -> - let content = if accept == OpenMetrics then hdrContentTypeOpenMetrics else hdrContentTypeText + let content = if accept == OpenMetrics then hdrContentTypeOpenMetrics else hdrContentTypePrometheus in responseMessage withBody content <$> getCurrentExposition <*> epochTime where withBody = method == GET -hdrContentType :: [ByteString] -> Builder -hdrContentType = mconcat . ("Content-Type: " :) . intersperse (char8 ';') . map byteString - -hdrContentTypeText, hdrContentTypeOpenMetrics :: Builder -hdrContentTypeText = hdrContentType ["text/plain", "charset=utf-8"] -hdrContentTypeOpenMetrics = hdrContentType ["application/openmetrics-text", "version=1.0.0", "charset=utf-8"] +hdrContentTypeText, hdrContentTypePrometheus, hdrContentTypeOpenMetrics :: Builder +hdrContentTypeText = "Content-Type: text/plain;charset=utf-8" +hdrContentTypePrometheus = "Content-Type: text/plain;version=0.0.4;charset=utf-8" +hdrContentTypeOpenMetrics = "Content-Type: application/openmetrics-text;version=1.0.0;charset=utf-8" hdrContentLength :: Int64 -> Builder hdrContentLength len = "Content-Length: " <> int64Dec len diff --git a/trace-dispatcher/src/Cardano/Logging/Tracer/Standard.hs b/trace-dispatcher/src/Cardano/Logging/Tracer/Standard.hs index 85fc3cfcd29..89be103f00d 100644 --- a/trace-dispatcher/src/Cardano/Logging/Tracer/Standard.hs +++ b/trace-dispatcher/src/Cardano/Logging/Tracer/Standard.hs @@ -11,10 +11,11 @@ import Cardano.Logging.Utils (threadLabelMe) import Control.Concurrent.Async import Control.Concurrent.Chan.Unagi.Bounded +import Control.Exception (BlockedIndefinitelyOnMVar (..), handle) import Control.Monad (forever, when) import Control.Monad.IO.Class import qualified Control.Tracer as T -import Data.IORef (IORef, modifyIORef', newIORef, readIORef) +import Data.IORef import Data.Maybe (isNothing) import Data.Text (Text) import qualified Data.Text.IO as TIO @@ -28,8 +29,8 @@ newtype StandardTracerState = StandardTracerState { emptyStandardTracerState :: StandardTracerState emptyStandardTracerState = StandardTracerState Nothing --- | It is mandatory to construct only one standard tracer in any application! --- Throwing away a standard tracer and using a new one will result in an exception +-- | The standardTracer handles stdout logging in a thread-safe manner. +-- It is strongly advised to construct only one standardTracer for any application. standardTracer :: forall m. (MonadIO m) => m (Trace m FormattedMessage) standardTracer = do @@ -41,17 +42,17 @@ standardTracer = do -> LoggingContext -> Either TraceControl FormattedMessage -> m () - output stateRef LoggingContext {} (Right (FormattedHuman _c msg)) = liftIO $ do + output stateRef LoggingContext{} (Right (FormattedHuman _c msg)) = liftIO $ do st <- readIORef stateRef case stRunning st of Just (inChannel, _, _) -> writeChan inChannel msg Nothing -> pure () - output stateRef LoggingContext {} (Right (FormattedMachine msg)) = liftIO $ do + output stateRef LoggingContext{} (Right (FormattedMachine msg)) = liftIO $ do st <- readIORef stateRef case stRunning st of Just (inChannel, _, _) -> writeChan inChannel msg Nothing -> pure () - output stateRef LoggingContext {} (Left TCReset) = liftIO $ do + output stateRef LoggingContext{} (Left TCReset) = liftIO $ do st <- readIORef stateRef case stRunning st of Nothing -> when (isNothing $ stRunning st) $ @@ -69,13 +70,14 @@ startStdoutThread stateRef = do (inChan, outChan) <- newChan 2048 as <- async $ threadLabelMe "StdoutTrace" >> stdoutThread outChan link as - modifyIORef' stateRef (\ st -> - st {stRunning = Just (inChan, outChan, as)}) + atomicWriteIORef stateRef $ StandardTracerState (Just (inChan, outChan, as)) -- | The new thread, which does the actual write from the queue. --- runs forever, and never returns +-- Will safely terminate when all producers have gone out of scope. stdoutThread :: OutChan Text -> IO () -stdoutThread outChan = forever $ do - readChan outChan - >>= TIO.putStrLn - hFlush stdout +stdoutThread outChan = + handle (\BlockedIndefinitelyOnMVar -> pure ()) $ + forever $ do + readChan outChan + >>= TIO.putStrLn + hFlush stdout diff --git a/trace-dispatcher/src/Cardano/Logging/Types.hs b/trace-dispatcher/src/Cardano/Logging/Types.hs index dc7f1cef830..59a18ba73b4 100644 --- a/trace-dispatcher/src/Cardano/Logging/Types.hs +++ b/trace-dispatcher/src/Cardano/Logging/Types.hs @@ -3,6 +3,7 @@ {-# LANGUAGE DerivingStrategies #-} {-# LANGUAGE GADTs #-} {-# LANGUAGE GeneralizedNewtypeDeriving #-} +{-# LANGUAGE MultiWayIf #-} {-# LANGUAGE RankNTypes #-} {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE ScopedTypeVariables #-} @@ -56,15 +57,16 @@ module Cardano.Logging.Types ( , TraceObject(..) , PreFormatted(..) , HowToConnect(..) - , howToConnectString ) where - import Codec.Serialise (Serialise (..)) +import Control.Applicative ((<|>)) +import Control.DeepSeq (NFData) import qualified Control.Tracer as T import qualified Data.Aeson as AE -import qualified Data.Aeson.Encoding as AE +import qualified Data.Aeson.Types as AE (Parser) import Data.Bool (bool) +import Data.ByteString (ByteString) import qualified Data.HashMap.Strict as HM import Data.IORef import Data.Kind (Type) @@ -72,9 +74,8 @@ import Data.Map.Strict (Map) import qualified Data.Map.Strict as Map import Data.Set (Set) import qualified Data.Set as Set -import Data.Text as T (Text, intercalate, null, pack, singleton, unpack, words) -import Data.Text.Lazy (toStrict) -import Data.Text.Lazy.Encoding (decodeUtf8) +import Data.Text as T (Text, breakOnEnd, intercalate, null, pack, singleton, unpack, + unsnoc, words) import Data.Text.Read as T (decimal) import Data.Time (UTCTime) import Data.Word (Word16) @@ -148,39 +149,24 @@ nsToText (Namespace ns1 ns2) = intercalate "." (ns1 ++ ns2) -- | Every message needs this to define how to represent itself class LogFormatting a where - -- | Machine readable representation with the possibility to represent - -- with different details based on the detail level. - -- No machine readable representation as default + -- | Machine readable representation with the possibility to represent with varying serialisations based on the detail level. + -- This will result in JSON formatted log output. + -- A `forMachine` implementation is required for any instance definition. forMachine :: DetailLevel -> a -> AE.Object - -- | Human readable representation. - -- No human representation is represented by the empty text - -- The default implementation returns no human representation + -- | Human-readable representation. + -- The empty text indicates there's no specific human-readable formatting for that type - this is the default implementation. + -- If however human-readble output is explicitly requested, e.g. by logs, the system will fall back to a JSON object + -- conforming to the `forMachine` definition, and rendering it as a value in `{"data": }`. + -- Leaving out `forHuman` in some instance definition will not lead to loss of log information that way. forHuman :: a -> Text forHuman _v = "" -- | Metrics representation. - -- No metrics by default + -- The default indicates that no metric is based on trace occurrences of that type. asMetrics :: a -> [Metric] asMetrics _v = [] - -- | A quick drop-in for forHuman to re-use the existing JSON serialization on normal detail level - -- You can safely use `forHuman = forHumanFromMachine` in instance definitions. - forHumanFromMachine :: a -> Text - forHumanFromMachine = - toStrict . decodeUtf8 . AE.encodingToLazyByteString . AE.toEncoding . forMachine DNormal - - -- | Yields the JSON serialization as Text if no human-readable representation is defined. - -- CAUTION: It is *NOT* safe to use it as a drop-in like forHumanFromMachine above - this leads to a stack overflow. - -- It's only meant to ease forHuman definitions using values of a distinct type such as: - -- `forHuman (ChainDB.PipeliningEvent ev') = forHumanFromMachine ev'` - -- In the future, the type should probably change to sth. akin to LogFormatting b => (a == b) ~ 'False => a -> b -> Text - -- to guard against that misuse. - forHumanOrMachine :: a -> Text - forHumanOrMachine v = - case forHuman v of - "" -> forHumanFromMachine v - s -> s class MetaTrace a where namespaceFor :: a -> Namespace a @@ -359,16 +345,16 @@ data FormattedMessage = | FormattedMachine Text | FormattedMetrics [Metric] | FormattedForwarder TraceObject + | FormattedCBOR ByteString deriving stock (Eq, Show) data PreFormatted = PreFormatted { - pfForHuman :: !(Maybe Text) - , pfForMachine :: !AE.Encoding - , pfNamespace :: ![Text] - , pfTimestamp :: !Text - , pfTime :: !UTCTime - , pfThreadId :: !Text + pfTime :: !UTCTime + , pfNamespace :: !Text + , pfThreadId :: !Text + , pfForHuman :: !(Maybe Text) + , pfForMachineObject :: AE.Object } -- | Used as interface object for ForwarderTracer @@ -634,9 +620,42 @@ type HowToConnect :: Type data HowToConnect = LocalPipe !FilePath -- ^ Local pipe (UNIX or Windows). | RemoteSocket !Host !Port -- ^ Remote socket (host and port). - deriving stock (Eq, Show, Generic) - -howToConnectString :: HowToConnect -> String -howToConnectString = \case - LocalPipe pipe -> pipe - RemoteSocket host port -> T.unpack host ++ ":" ++ show port + deriving stock (Eq, Generic) + deriving anyclass (NFData) + +instance Show HowToConnect where + show = \case + LocalPipe pipe -> pipe + RemoteSocket host port -> T.unpack host ++ ":" ++ show port + +instance AE.ToJSON HowToConnect where + toJSON = AE.toJSON . show + toEncoding = AE.toEncoding . show + +-- first try to host:port, and if that fails revert to parsing any +-- string literal and assume it is a localpipe. +instance AE.FromJSON HowToConnect where + parseJSON = AE.withText "HowToConnect" $ \t -> + (uncurry RemoteSocket <$> parseHostPort t) + <|> ( LocalPipe <$> parseLocalPipe t) + +parseLocalPipe :: Text -> AE.Parser FilePath +parseLocalPipe t + | T.null t = fail "parseLocalPipe: empty Text" + | otherwise = pure $ T.unpack t + +parseHostPort :: Text -> AE.Parser (Text, Word16) +parseHostPort t + | T.null t + = fail "parseHostPort: empty Text" + | otherwise + = let + (host_, portText) = T.breakOnEnd ":" t + host = maybe "" fst (T.unsnoc host_) + in if + | T.null host -> fail "parseHostPort: Empty host or no colon found." + | T.null portText -> fail "parseHostPort: Empty port." + | Right (port, remainder) <- T.decimal portText + , T.null remainder + , 0 <= port, port <= 65535 -> pure (host, port) + | otherwise -> fail "parseHostPort: Non-numeric port or value out of range." diff --git a/trace-dispatcher/src/Cardano/Logging/Types/TraceMessage.hs b/trace-dispatcher/src/Cardano/Logging/Types/TraceMessage.hs new file mode 100644 index 00000000000..6310baf2044 --- /dev/null +++ b/trace-dispatcher/src/Cardano/Logging/Types/TraceMessage.hs @@ -0,0 +1,83 @@ + +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE RecordWildCards #-} + +{-# OPTIONS_GHC -Wno-orphans #-} + +module Cardano.Logging.Types.TraceMessage + ( TraceMessage (..) + ) where + +import Cardano.Logging.Types + +import Codec.CBOR.JSON +import Codec.Serialise (Serialise (..)) +import Data.Aeson as AE hiding (decode, encode) +import Data.Text (Text) +import Data.Time.Clock (UTCTime) + + +-- | base for a machine readable trace message (JSON or CBOR), with metadata, and enclosed payload data from the trace itself. +data TraceMessage = TraceMessage + { tmsgAt :: !UTCTime + , tmsgNS :: !Text + , tmsgData :: !AE.Object + , tmsgSev :: !SeverityS + , tmsgThread :: !Text + , tmsgHost :: !Text + } + deriving Show + +instance Serialise AE.Object where + encode = encodeValue . Object + decode = decodeValue True >>= \case + Object o -> pure o + x -> fail $ "decode(TraceMessage): expected JSON object, got: " ++ show x + + +-- Serialisations are hand-rolled for higher degree of stability, and making them transparent. +instance Serialise TraceMessage where + encode TraceMessage{..} = + encode tmsgAt + <> encode tmsgNS + <> encode tmsgSev + <> encode tmsgData + <> encode tmsgThread + <> encode tmsgHost + + decode = do + tmsgAt <- decode + tmsgNS <- decode + tmsgSev <- decode + tmsgData <- decode + tmsgThread <- decode + tmsgHost <- decode + pure TraceMessage{..} + + +instance ToJSON TraceMessage where + toJSON TraceMessage{..} = AE.object + [ "at" .= tmsgAt + , "ns" .= tmsgNS + , "data" .= tmsgData + , "sev" .= tmsgSev + , "thread" .= tmsgThread + , "host" .= tmsgHost + ] + toEncoding TraceMessage{..} = AE.pairs $ + "at" .= tmsgAt + <> "ns" .= tmsgNS + <> "data" .= tmsgData + <> "sev" .= tmsgSev + <> "thread" .= tmsgThread + <> "host" .= tmsgHost + +instance FromJSON TraceMessage where + parseJSON = AE.withObject "TraceMessage" $ \v -> do + tmsgAt <- v .: "at" + tmsgNS <- v .: "ns" + tmsgData <- v .: "data" + tmsgSev <- v .: "sev" + tmsgThread <- v .: "thread" + tmsgHost <- v .: "host" + pure TraceMessage{..} diff --git a/trace-dispatcher/src/Cardano/Logging/Utils.hs b/trace-dispatcher/src/Cardano/Logging/Utils.hs index 0f116b5c721..2630da48f05 100644 --- a/trace-dispatcher/src/Cardano/Logging/Utils.hs +++ b/trace-dispatcher/src/Cardano/Logging/Utils.hs @@ -1,11 +1,14 @@ +{-# LANGUAGE NumericUnderscores #-} + module Cardano.Logging.Utils ( module Cardano.Logging.Utils ) where -import Cardano.Logging.Types (HowToConnect) + import Control.Concurrent (threadDelay) -import Control.Exception (SomeAsyncException (..), fromException, tryJust) -import Control.Tracer (stdoutTracer, traceWith) +import Control.Concurrent.Async (concurrently_) +import Control.Exception (SomeAsyncException (..), SomeException, fromException, tryJust) +import Data.IORef import qualified Data.Text as T import qualified Data.Text.Lazy as TL (toStrict) import qualified Data.Text.Lazy.Builder as T (toLazyText) @@ -14,26 +17,40 @@ import qualified Data.Text.Lazy.Builder.RealFloat as T (realFloat) import GHC.Conc (labelThread, myThreadId) --- | Run monadic action in a loop. If there's an exception, it will re-run --- the action again, after pause that grows. -runInLoop :: IO () -> HowToConnect -> Word -> Word -> IO () -runInLoop action howToConnect prevDelayInSecs maxReconnectDelay = - tryJust excludeAsyncExceptions action >>= \case - Left e -> do - logTrace $ "connection with " <> show howToConnect <> " failed: " <> show e - threadDelay . fromIntegral $ currentDelayInSecs * 1000000 - runInLoop action howToConnect currentDelayInSecs maxReconnectDelay - Right _ -> return () - where - excludeAsyncExceptions e = - case fromException e of - Just SomeAsyncException {} -> Nothing - _ -> Just e - - logTrace = traceWith stdoutTracer - - currentDelayInSecs = - min (prevDelayInSecs * 2) maxReconnectDelay +-- | Run an IO action which may throw an exception in a loop. +-- On exception, the action will be re-run after a pause. +-- That pause doubles which each exception, but is reset when the action runs long enough. +runInLoop :: IO () -> (SomeException -> IO ()) -> Word -> Word -> IO () +runInLoop action handleInterruption initialDelay maxDelay + | initialDelay == 0 = runInLoop action handleInterruption 1 maxDelay + | maxDelay < initialDelay = runInLoop action handleInterruption initialDelay initialDelay + | otherwise = newIORef (fromIntegral initialDelay) >>= go + where + go :: IORef Int -> IO () + go currentDelay = + tryJust excludeAsyncExceptions (actionResettingDelay currentDelay) >>= \case + Left e -> do + handleInterruption e + waitForSecs <- atomicModifyIORef' currentDelay bumpDelay + threadDelay $ 1_000_000 * waitForSecs + go currentDelay + Right _ -> return () + + -- if the action runs at least maxDelay seconds, the pause is reset + actionResettingDelay currentDelay = concurrently_ action $ do + threadDelay $ fromIntegral $ 1_000_000 * maxDelay + atomicWriteIORef currentDelay $ fromIntegral initialDelay + + excludeAsyncExceptions e = + case fromException e of + Just SomeAsyncException{} -> Nothing + _ -> Just e + + bumpDelay current = + ( min (current * 2) (fromIntegral maxDelay) + , current + ) + -- | Convenience function for a Show instance to be converted to text immediately {-# INLINE showT #-} diff --git a/trace-dispatcher/test/Cardano/Logging/Test/Tracer.hs b/trace-dispatcher/test/Cardano/Logging/Test/Tracer.hs index 514a3c9db98..541e50ac8ad 100644 --- a/trace-dispatcher/test/Cardano/Logging/Test/Tracer.hs +++ b/trace-dispatcher/test/Cardano/Logging/Test/Tracer.hs @@ -4,19 +4,19 @@ module Cardano.Logging.Test.Tracer ( testTracer , formattedMsgAsText - , LoggingMessage (..) , testLoggingMessageEq , testLoggingMessagesEq ) where import Cardano.Logging +import Cardano.Logging.Types.TraceMessage import Control.Monad.IO.Class -import Data.Aeson (FromJSON (..), Object, decodeStrict, withObject, (.:)) +import Data.Aeson (decodeStrict) +import Data.Function (on) import Data.IORef import Data.Text (Text, pack, unpack) import Data.Text.Encoding (encodeUtf8) -import Data.Time (UTCTime) testTracer :: MonadIO m @@ -39,43 +39,31 @@ formattedMsgAsText (FormattedHuman _ text) = text formattedMsgAsText (FormattedMachine text) = text formattedMsgAsText (FormattedMetrics metrics) = pack (show metrics) formattedMsgAsText (FormattedForwarder traceObj) = toMachine traceObj - -data LoggingMessage = LoggingMessage { - at :: UTCTime - , ns :: Text - , dataX :: Object - , sev :: SeverityS - , thread :: Text - , host :: Text - } deriving (Eq, Ord, Show) - -instance FromJSON LoggingMessage where - parseJSON = withObject "LoggingMessage" $ \v -> LoggingMessage - <$> v .: "at" - <*> v .: "ns" - <*> v .: "data" - <*> v .: "sev" - <*> v .: "thread" - <*> v .: "host" +formattedMsgAsText (FormattedCBOR _) = error "FormattedMessage.FormattedCBOR currently has no Text representation" testLoggingMessageEq :: Text -> Text -> IO Bool testLoggingMessageEq t1 t2 = - let lm1 = (decodeStrict . encodeUtf8) t1 :: Maybe LoggingMessage - lm2 = (decodeStrict . encodeUtf8) t2 :: Maybe LoggingMessage + let lm1 = (decodeStrict . encodeUtf8) t1 :: Maybe TraceMessage + lm2 = (decodeStrict . encodeUtf8) t2 :: Maybe TraceMessage in case (lm1, lm2) of - (Just (LoggingMessage _at1 ns1 dataX1 sev1 _thread1 _host1), - Just (LoggingMessage _at2 ns2 dataX2 sev2 _thread2 _host2)) -> - let res = ns1 == ns2 && dataX1 == dataX2 && sev1 == sev2 - in if not res - then do - putStrLn ("Failed ns1: " ++ show ns1 ++ " ns2: " ++ show ns2 ++ - " dataX1: " ++ show dataX1 ++ " dataX2: " ++ show dataX2 ++ - " sev1: " ++ show sev1 ++ " sev2: " ++ show sev2) - pure False - else pure True + (Just parse1, Just parse2) -> + let + constraints = + [ (==) `on` tmsgNS + , (==) `on` tmsgData + , (==) `on` tmsgSev + ] + allConstraintsHold = all (\check -> check parse1 parse2) constraints + in if not allConstraintsHold + then do + putStrLn $ "Failed ns1: " ++ show (tmsgNS parse1) ++ " ns2: " ++ show (tmsgNS parse2) ++ + " / data1: " ++ show (tmsgData parse1) ++ " data2: " ++ show (tmsgData parse1) ++ + " / sev1: " ++ show (tmsgSev parse1) ++ " sev2: " ++ show (tmsgSev parse2) + pure False + else pure True _ -> do - putStrLn ("Failed t1:" ++ unpack t1 ++ " t2: " ++ unpack t2 - ++ " lm1 " ++ show lm1 ++ " lm2 " ++ show lm2) + putStrLn $ "Failed t1:" ++ unpack t1 ++ " t2: " ++ unpack t2 ++ + " / lm1 " ++ show lm1 ++ " lm2 " ++ show lm2 pure False testLoggingMessagesEq :: [Text] -> [Text] -> IO Bool diff --git a/trace-dispatcher/trace-dispatcher.cabal b/trace-dispatcher/trace-dispatcher.cabal index 6b6da149118..efca9608ee7 100644 --- a/trace-dispatcher/trace-dispatcher.cabal +++ b/trace-dispatcher/trace-dispatcher.cabal @@ -1,7 +1,7 @@ cabal-version: 3.0 name: trace-dispatcher -version: 2.10.0 +version: 2.11.0 synopsis: Tracers for Cardano description: Package for development of simple and efficient tracers based on the arrow based contra-tracer package @@ -61,6 +61,7 @@ library Cardano.Logging.Tracer.Forward Cardano.Logging.Tracer.Composed Cardano.Logging.Types + Cardano.Logging.Types.TraceMessage Cardano.Logging.Types.NodeInfo Cardano.Logging.Types.NodePeers Cardano.Logging.Types.NodeStartupInfo @@ -73,6 +74,8 @@ library , aeson-pretty , async , bytestring + , cborg + , cborg-json , containers , contra-tracer , deepseq diff --git a/trace-forward/CHANGELOG.md b/trace-forward/CHANGELOG.md index fa1d24e9086..e5240f201ab 100644 --- a/trace-forward/CHANGELOG.md +++ b/trace-forward/CHANGELOG.md @@ -1,8 +1,10 @@ # ChangeLog -## NEXT - Nov 2025 +## 2.4.0 - Nov 2025 -* Replaced both `disconnectedQueueSize` and `connectedQueueSize` with `queueSize` (See #6361 for details). +* Refactor `writeToSink` and `readFromSink` to simplify STM usage +* Drop incongruous logic of switching queue capacity from `ForwardSink`, along with related fields and code +* Provide `InitForwardingConfig` config record type for `initForwarding` and `initForwardingDelayed` ## 2.3.1 - Oct 2025 diff --git a/trace-forward/src/Trace/Forward/Forwarding.hs b/trace-forward/src/Trace/Forward/Forwarding.hs index 67102ca3757..fae115607c2 100644 --- a/trace-forward/src/Trace/Forward/Forwarding.hs +++ b/trace-forward/src/Trace/Forward/Forwarding.hs @@ -1,14 +1,17 @@ {-# LANGUAGE BlockArguments #-} - {-# LANGUAGE DataKinds #-} {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE PackageImports #-} +{-# LANGUAGE RecordWildCards #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} {-# LANGUAGE ViewPatterns #-} +{-# OPTIONS_GHC -Wno-partial-fields #-} + module Trace.Forward.Forwarding - ( initForwarding + ( InitForwardingConfig(..) + , initForwarding , initForwardingDelayed ) where @@ -26,22 +29,22 @@ import Ouroboros.Network.Protocol.Handshake.Codec (cborTermVersionData import Ouroboros.Network.Protocol.Handshake.Type (Handshake) import Ouroboros.Network.Protocol.Handshake.Version (acceptableVersion, queryVersion, simpleSingletonVersions) -import Ouroboros.Network.Snocket (MakeBearer, Snocket, localAddressFromPath, localSnocket, - makeLocalBearer, LocalAddress, socketSnocket, makeSocketBearer, LocalSocket) -import Ouroboros.Network.Socket (ConnectToArgs (..), - HandshakeCallbacks (..), SomeResponderApplication (..), - connectToNode, nullNetworkConnectTracers) import qualified Ouroboros.Network.Server.Simple as Server +import Ouroboros.Network.Snocket (LocalAddress, LocalSocket, MakeBearer, Snocket, + localAddressFromPath, localSnocket, makeLocalBearer, makeSocketBearer, + socketSnocket) +import Ouroboros.Network.Socket (ConnectToArgs (..), HandshakeCallbacks (..), + SomeResponderApplication (..), connectToNode, nullNetworkConnectTracers) import Codec.CBOR.Term (Term) import Control.Concurrent.Async (async, wait) -import Control.Exception (throwIO) +import Control.Exception (SomeException, throwIO) import Control.Monad.IO.Class import "contra-tracer" Control.Tracer (Tracer, contramap, nullTracer, stdoutTracer) import qualified Data.ByteString.Lazy as LBS import Data.Functor import Data.List.NonEmpty (NonEmpty ((:|))) -import Data.Maybe (isNothing) +import Data.Maybe (fromMaybe) import qualified Data.Text as Text import Data.Void (Void, absurd) import Data.Word (Word16) @@ -61,15 +64,37 @@ import Trace.Forward.Utils.ForwardSink (ForwardSink) import Trace.Forward.Utils.TraceObject import Trace.Forward.Utils.Version + +-- | Config record to initialise trace forwarding +data InitForwardingConfig + = -- | Only construct relevant values, but do not actually run forwarding + InitForwardingNone + | -- | Run forwarding with the provided settings + InitForwardingWith + { initNetworkMagic :: !NetworkMagic + -- ^ Forwarding is always tied to a singular networkId + , initEKGStore :: !(Maybe EKG.Store) + -- ^ A metrics store to be forwarded (optional) + , initHowToConnect :: !HowToConnect + -- ^ A LocalPipe or RemoteSocket + , initForwarderMode :: !ForwarderMode + -- ^ Run as Initiator or Responder + , initOnForwardInterruption :: !(Maybe (SomeException -> IO ())) + -- ^ Optional handler when forwarding connection is interrupted (may be temporary or permanent) + -- default: no action + , initOnQueueOverflow :: !(Maybe ([TraceObject] -> IO ())) + -- ^ Optional handler when forwarding queue overflows (argument are objects dropped from queue) + -- default: print one-liner to stderr, indicating object count and timestamps of first and last object + } + + initForwarding :: forall m. (MonadIO m) => IOManager -> TraceOptionForwarder - -> NetworkMagic - -> Maybe EKG.Store - -> Maybe (HowToConnect, ForwarderMode) + -> InitForwardingConfig -> m (ForwardSink TraceObject, DataPointStore) -initForwarding iomgr config magic ekgStore tracerSocketMode = do - (a, b, kickoffForwarder) <- initForwardingDelayed iomgr config magic ekgStore tracerSocketMode +initForwarding iomgr config forwarding = do + (a, b, kickoffForwarder) <- initForwardingDelayed iomgr config forwarding liftIO kickoffForwarder pure (a, b) @@ -79,38 +104,34 @@ initForwardingDelayed :: forall m. () => MonadIO m => IOManager -> TraceOptionForwarder - -> NetworkMagic - -> Maybe EKG.Store - -> Maybe (HowToConnect, ForwarderMode) + -> InitForwardingConfig -> m (ForwardSink TraceObject, DataPointStore, IO ()) -initForwardingDelayed iomgr config magic ekgStore tracerSocketMode = liftIO $ do - let ignoreOverflow, onOverflow :: [TraceObject] -> IO () - ignoreOverflow _ = - pure () - onOverflow | isNothing tracerSocketMode = ignoreOverflow - | otherwise = handleOverflow +initForwardingDelayed iomgr config forwarding = liftIO $ do + let onOverflow :: [TraceObject] -> IO () + onOverflow = case forwarding of + InitForwardingNone -> const $ pure () + InitForwardingWith{initOnQueueOverflow = Just handler} -> handler + InitForwardingWith{initOnQueueOverflow = Nothing} -> handleOverflow forwardSink <- initForwardSink tfConfig onOverflow dpStore <- initDataPointStore let kickoffForwarder = launchForwarders iomgr - magic + forwarding ekgConfig tfConfig dpfConfig - ekgStore forwardSink dpStore - tracerSocketMode maxReconnectDelay pure (forwardSink, dpStore, kickoffForwarder) where endpoint :: EKGF.HowToConnect endpoint = - case tracerSocketMode of - Nothing -> EKGF.LocalPipe "" - Just (LocalPipe str, _mode) -> EKGF.LocalPipe str - Just (RemoteSocket host port, _mode) -> EKGF.RemoteSocket host port + case forwarding of + InitForwardingNone -> EKGF.LocalPipe "" + InitForwardingWith{initHowToConnect = LocalPipe str} -> EKGF.LocalPipe str + InitForwardingWith{initHowToConnect = RemoteSocket host port} -> EKGF.RemoteSocket host port queueSize = tofQueueSize config verbosity = tofVerbosity config maxReconnectDelay = tofMaxReconnectDelay config @@ -157,39 +178,37 @@ handleOverflow (msg : msgs) = launchForwarders :: IOManager - -> NetworkMagic + -> InitForwardingConfig -> EKGF.ForwarderConfiguration -> TF.ForwarderConfiguration TraceObject -> DPF.ForwarderConfiguration - -> Maybe EKG.Store -> ForwardSink TraceObject -> DataPointStore - -> Maybe (HowToConnect, ForwarderMode) -> Word -> IO () -launchForwarders iomgr magic +launchForwarders iomgr forwarding ekgConfig tfConfig dpfConfig - ekgStore sink dpStore tracerSocketMode maxReconnectDelay = - -- If 'tracerSocketMode' is not specified, it's impossible to establish - -- network connection with acceptor application (for example, 'cardano-tracer'). + sink dpStore maxReconnectDelay = + -- If InitForwardingNone is specified, it's impossible to establish + -- a connection with an acceptor application (for example, 'cardano-tracer'). -- In this case, we should not launch forwarders. - case tracerSocketMode of - Nothing -> return () - Just (socketPath, mode) -> + case forwarding of + InitForwardingNone -> return () + InitForwardingWith{..} -> void . async $ runInLoop (launchForwardersViaLocalSocket iomgr - magic - socketPath - mode + initNetworkMagic + initHowToConnect + initForwarderMode ekgConfig tfConfig dpfConfig sink - ekgStore + initEKGStore dpStore) - socketPath + (fromMaybe (const $ pure ()) initOnForwardInterruption) 1 maxReconnectDelay diff --git a/trace-forward/trace-forward.cabal b/trace-forward/trace-forward.cabal index d37b2ff4112..992b26560d5 100644 --- a/trace-forward/trace-forward.cabal +++ b/trace-forward/trace-forward.cabal @@ -1,7 +1,7 @@ cabal-version: 3.0 name: trace-forward -version: 2.3.1 +version: 2.4.0 synopsis: The forwarding protocols library for cardano node. description: The library providing typed protocols for forwarding different information from the cardano node to an external application.