From 92312801f168bea810c1e2af39b1726729761bf9 Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 21 Aug 2025 14:37:53 +0100 Subject: [PATCH 01/88] Support structured logs Signed-off-by: JCW --- cmake/RippledCore.cmake | 9 +- cmake/RippledInstall.cmake | 1 + include/xrpl/basics/Log.h | 21 +- include/xrpl/beast/utility/Journal.h | 253 ++++++++++++-- include/xrpl/logging/JsonLogs.h | 223 +++++++++++++ include/xrpl/resource/detail/Logic.h | 30 +- include/xrpl/server/detail/BasePeer.h | 16 +- include/xrpl/telemetry/JsonLogs.h | 221 ++++++++++++ src/libxrpl/basics/Log.cpp | 79 +++-- src/libxrpl/beast/utility/beast_Journal.cpp | 61 +++- src/libxrpl/logging/JsonLogs.cpp | 136 ++++++++ src/libxrpl/telemetry/JsonLogs.cpp | 131 ++++++++ src/test/csf/Peer.h | 5 +- src/tests/libxrpl/CMakeLists.txt | 2 + src/tests/libxrpl/basics/log.cpp | 169 ++++++++++ src/tests/libxrpl/telemetry/base64.cpp | 67 ++++ src/tests/libxrpl/telemetry/json_logs.cpp | 353 ++++++++++++++++++++ src/tests/libxrpl/telemetry/main.cpp | 2 + src/xrpld/app/consensus/RCLConsensus.cpp | 16 +- src/xrpld/app/consensus/RCLConsensus.h | 4 +- src/xrpld/app/main/Application.cpp | 23 +- src/xrpld/app/main/Application.h | 5 +- src/xrpld/app/main/Main.cpp | 9 + src/xrpld/app/tx/detail/Transactor.cpp | 7 +- src/xrpld/app/tx/detail/Transactor.h | 4 +- src/xrpld/core/Config.h | 14 + src/xrpld/core/ConfigSections.h | 1 + src/xrpld/core/detail/Config.cpp | 13 + src/xrpld/overlay/detail/ConnectAttempt.cpp | 3 +- src/xrpld/overlay/detail/ConnectAttempt.h | 1 - src/xrpld/overlay/detail/OverlayImpl.cpp | 11 +- src/xrpld/overlay/detail/OverlayImpl.h | 3 - src/xrpld/overlay/detail/PeerImp.cpp | 16 +- src/xrpld/overlay/detail/PeerImp.h | 21 +- 34 files changed, 1801 insertions(+), 129 deletions(-) create mode 100644 include/xrpl/logging/JsonLogs.h create mode 100644 include/xrpl/telemetry/JsonLogs.h create mode 100644 src/libxrpl/logging/JsonLogs.cpp create mode 100644 src/libxrpl/telemetry/JsonLogs.cpp create mode 100644 src/tests/libxrpl/basics/log.cpp create mode 100644 src/tests/libxrpl/telemetry/base64.cpp create mode 100644 src/tests/libxrpl/telemetry/json_logs.cpp create mode 100644 src/tests/libxrpl/telemetry/main.cpp diff --git a/cmake/RippledCore.cmake b/cmake/RippledCore.cmake index 481b6e3cea6..0e7ab8be211 100644 --- a/cmake/RippledCore.cmake +++ b/cmake/RippledCore.cmake @@ -85,8 +85,14 @@ target_link_libraries(xrpl.libxrpl.basics PUBLIC xrpl.libxrpl.beast) add_module(xrpl json) target_link_libraries(xrpl.libxrpl.json PUBLIC xrpl.libxrpl.basics) +add_module(xrpl telemetry) +target_link_libraries(xrpl.libxrpl.telemetry PUBLIC xrpl.libxrpl.json) + add_module(xrpl crypto) -target_link_libraries(xrpl.libxrpl.crypto PUBLIC xrpl.libxrpl.basics) +target_link_libraries(xrpl.libxrpl.crypto PUBLIC + xrpl.libxrpl.basics + xrpl.libxrpl.telemetry +) # Level 04 add_module(xrpl protocol) @@ -133,6 +139,7 @@ target_link_modules(xrpl PUBLIC beast crypto json + telemetry protocol resource server diff --git a/cmake/RippledInstall.cmake b/cmake/RippledInstall.cmake index 95c25a212f9..48fb0fcf954 100644 --- a/cmake/RippledInstall.cmake +++ b/cmake/RippledInstall.cmake @@ -16,6 +16,7 @@ install ( xrpl.libxrpl.beast xrpl.libxrpl.crypto xrpl.libxrpl.json + xrpl.libxrpl.telemetry xrpl.libxrpl.protocol xrpl.libxrpl.resource xrpl.libxrpl.ledger diff --git a/include/xrpl/basics/Log.h b/include/xrpl/basics/Log.h index 833907eb9c8..2c407dda171 100644 --- a/include/xrpl/basics/Log.h +++ b/include/xrpl/basics/Log.h @@ -167,6 +167,8 @@ class Logs beast::severities::Severity thresh_; File file_; bool silent_ = false; + static std::unique_ptr + globalLogAttributes_; public: Logs(beast::severities::Severity level); @@ -187,7 +189,10 @@ class Logs operator[](std::string const& name); beast::Journal - journal(std::string const& name); + journal( + std::string const& name, + std::unique_ptr attributes = + {}); beast::severities::Severity threshold() const; @@ -224,6 +229,20 @@ class Logs std::string const& partition, beast::severities::Severity startingLevel); + static void + setGlobalAttributes(std::unique_ptr + globalLogAttributes) + { + if (!globalLogAttributes_) + { + globalLogAttributes_ = std::move(globalLogAttributes); + } + else + { + globalLogAttributes_->combine(std::move(globalLogAttributes_)); + } + } + public: static LogSeverity fromSeverity(beast::severities::Severity level); diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 91ddf33d271..5942b8de095 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -22,6 +22,7 @@ #include +#include #include namespace beast { @@ -42,6 +43,9 @@ enum Severity { kDisabled, kNone = kDisabled }; + +std::string +to_string(Severity severity); } // namespace severities /** A generic endpoint for log messages. @@ -61,16 +65,68 @@ class Journal public: class Sink; + class StructuredJournalImpl; + + class StructuredLogAttributes; + private: // Severity level / threshold of a Journal message. using Severity = severities::Severity; + std::unique_ptr m_attributes; + + static StructuredJournalImpl* m_structuredJournalImpl; + // Invariant: m_sink always points to a valid Sink - Sink* m_sink; + Sink* m_sink = nullptr; public: //-------------------------------------------------------------------------- + static void + enableStructuredJournal(StructuredJournalImpl* impl) + { + m_structuredJournalImpl = impl; + } + + static bool + isStructuredJournalEnabled() + { + return m_structuredJournalImpl; + } + + class StructuredJournalImpl + { + public: + StructuredJournalImpl() = default; + StructuredJournalImpl(StructuredJournalImpl const&) = default; + virtual void + initMessageContext(std::source_location location) = 0; + virtual void + flush( + Sink* sink, + severities::Severity level, + std::string const& text, + StructuredLogAttributes* attributes) = 0; + virtual ~StructuredJournalImpl() = default; + }; + + class StructuredLogAttributes + { + public: + StructuredLogAttributes() = default; + StructuredLogAttributes(StructuredLogAttributes const&) = default; + virtual void + setModuleName(std::string const& name) = 0; + virtual std::unique_ptr + clone() const = 0; + virtual void + combine(std::unique_ptr const& attributes) = 0; + virtual void + combine(std::unique_ptr&& attributes) = 0; + virtual ~StructuredLogAttributes() = default; + }; + /** Abstraction for the underlying message destination. */ class Sink { @@ -150,16 +206,28 @@ class Journal { public: ScopedStream(ScopedStream const& other) - : ScopedStream(other.m_sink, other.m_level) + : ScopedStream( + other.m_attributes ? other.m_attributes->clone() : nullptr, + other.m_sink, + other.m_level) { } - ScopedStream(Sink& sink, Severity level); + ScopedStream( + std::unique_ptr attributes, + Sink& sink, + Severity level); template - ScopedStream(Stream const& stream, T const& t); + ScopedStream( + std::unique_ptr attributes, + Stream const& stream, + T const& t); - ScopedStream(Stream const& stream, std::ostream& manip(std::ostream&)); + ScopedStream( + std::unique_ptr attributes, + Stream const& stream, + std::ostream& manip(std::ostream&)); ScopedStream& operator=(ScopedStream const&) = delete; @@ -180,6 +248,7 @@ class Journal operator<<(T const& t) const; private: + std::unique_ptr m_attributes; Sink& m_sink; Severity const m_level; std::ostringstream mutable m_ostream; @@ -214,7 +283,11 @@ class Journal Constructor is inlined so checking active() very inexpensive. */ - Stream(Sink& sink, Severity level) : m_sink(sink), m_level(level) + Stream( + std::unique_ptr attributes, + Sink& sink, + Severity level) + : m_attributes(std::move(attributes)), m_sink(sink), m_level(level) { XRPL_ASSERT( m_level < severities::kDisabled, @@ -222,7 +295,11 @@ class Journal } /** Construct or copy another Stream. */ - Stream(Stream const& other) : Stream(other.m_sink, other.m_level) + Stream(Stream const& other) + : Stream( + other.m_attributes ? other.m_attributes->clone() : nullptr, + other.m_sink, + other.m_level) { } @@ -269,6 +346,7 @@ class Journal /** @} */ private: + std::unique_ptr m_attributes; Sink& m_sink; Severity m_level; }; @@ -287,9 +365,90 @@ class Journal /** Journal has no default constructor. */ Journal() = delete; + [[deprecated]] + Journal(Journal const& other) + : Journal(other, nullptr) + { + } + + Journal( + Journal const& other, + std::unique_ptr attributes) + : m_sink(other.m_sink) + { + if (attributes) + { + m_attributes = std::move(attributes); + } + if (other.m_attributes) + { + if (m_attributes) + { + m_attributes->combine(other.m_attributes); + } + else + { + m_attributes = other.m_attributes->clone(); + } + } + } + + Journal( + Journal&& other, + std::unique_ptr attributes = {}) noexcept + : m_sink(other.m_sink) + { + if (attributes) + { + m_attributes = std::move(attributes); + } + if (other.m_attributes) + { + if (m_attributes) + { + m_attributes->combine(std::move(other.m_attributes)); + } + else + { + m_attributes = std::move(other.m_attributes); + } + } + } + /** Create a journal that writes to the specified sink. */ - explicit Journal(Sink& sink) : m_sink(&sink) + Journal( + Sink& sink, + std::string const& name = {}, + std::unique_ptr attributes = {}) + : m_sink(&sink) + { + if (attributes) + { + m_attributes = std::move(attributes); + m_attributes->setModuleName(name); + } + } + + Journal& + operator=(Journal const& other) + { + m_sink = other.m_sink; + if (other.m_attributes) + { + m_attributes = other.m_attributes->clone(); + } + return *this; + } + + Journal& + operator=(Journal&& other) noexcept { + m_sink = other.m_sink; + if (other.m_attributes) + { + m_attributes = std::move(other.m_attributes); + } + return *this; } /** Returns the Sink associated with this Journal. */ @@ -303,7 +462,8 @@ class Journal Stream stream(Severity level) const { - return Stream(*m_sink, level); + return Stream( + m_attributes ? m_attributes->clone() : nullptr, *m_sink, level); } /** Returns `true` if any message would be logged at this severity level. @@ -319,39 +479,81 @@ class Journal /** Severity stream access functions. */ /** @{ */ Stream - trace() const + trace(std::source_location location = std::source_location::current()) const { - return {*m_sink, severities::kTrace}; + if (m_structuredJournalImpl) + { + m_structuredJournalImpl->initMessageContext(location); + } + return { + m_attributes ? m_attributes->clone() : nullptr, + *m_sink, + severities::kTrace}; } Stream - debug() const + debug(std::source_location location = std::source_location::current()) const { - return {*m_sink, severities::kDebug}; + if (m_structuredJournalImpl) + { + m_structuredJournalImpl->initMessageContext(location); + } + return { + m_attributes ? m_attributes->clone() : nullptr, + *m_sink, + severities::kDebug}; } Stream - info() const + info(std::source_location location = std::source_location::current()) const { - return {*m_sink, severities::kInfo}; + if (m_structuredJournalImpl) + { + m_structuredJournalImpl->initMessageContext(location); + } + return { + m_attributes ? m_attributes->clone() : nullptr, + *m_sink, + severities::kInfo}; } Stream - warn() const + warn(std::source_location location = std::source_location::current()) const { - return {*m_sink, severities::kWarning}; + if (m_structuredJournalImpl) + { + m_structuredJournalImpl->initMessageContext(location); + } + return { + m_attributes ? m_attributes->clone() : nullptr, + *m_sink, + severities::kWarning}; } Stream - error() const + error(std::source_location location = std::source_location::current()) const { - return {*m_sink, severities::kError}; + if (m_structuredJournalImpl) + { + m_structuredJournalImpl->initMessageContext(location); + } + return { + m_attributes ? m_attributes->clone() : nullptr, + *m_sink, + severities::kError}; } Stream - fatal() const + fatal(std::source_location location = std::source_location::current()) const { - return {*m_sink, severities::kFatal}; + if (m_structuredJournalImpl) + { + m_structuredJournalImpl->initMessageContext(location); + } + return { + m_attributes ? m_attributes->clone() : nullptr, + *m_sink, + severities::kFatal}; } /** @} */ }; @@ -368,8 +570,11 @@ static_assert(std::is_nothrow_destructible::value == true, ""); //------------------------------------------------------------------------------ template -Journal::ScopedStream::ScopedStream(Journal::Stream const& stream, T const& t) - : ScopedStream(stream.sink(), stream.level()) +Journal::ScopedStream::ScopedStream( + std::unique_ptr attributes, + Stream const& stream, + T const& t) + : ScopedStream(std::move(attributes), stream.sink(), stream.level()) { m_ostream << t; } @@ -388,7 +593,7 @@ template Journal::ScopedStream Journal::Stream::operator<<(T const& t) const { - return ScopedStream(*this, t); + return {m_attributes ? m_attributes->clone() : nullptr, *this, t}; } namespace detail { diff --git a/include/xrpl/logging/JsonLogs.h b/include/xrpl/logging/JsonLogs.h new file mode 100644 index 00000000000..1eb2ba8c8a2 --- /dev/null +++ b/include/xrpl/logging/JsonLogs.h @@ -0,0 +1,223 @@ +//------------------------------------------------------------------------------ +/* + This file is part of Beast: https://github.com/vinniefalco/Beast + Copyright 2013, Vinnie Falco + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_LOGGING_STRUCTUREDJOURNAL_H_INCLUDED +#define RIPPLE_LOGGING_STRUCTUREDJOURNAL_H_INCLUDED + +#include +#include +#include +#include + +#include +#include +#include + +namespace ripple { +namespace log { + +template +class LogParameter +{ +public: + template + LogParameter(char const* name, TArg&& value) + : name_(name), value_(std::forward(value)) + { + } + +private: + char const* name_; + T value_; + + template + friend std::ostream& + operator<<(std::ostream& os, LogParameter const&); +}; + +template +class LogField +{ +public: + template + LogField(char const* name, TArg&& value) + : name_(name), value_(std::forward(value)) + { + } + +private: + char const* name_; + T value_; + + template + friend std::ostream& + operator<<(std::ostream& os, LogField const&); +}; + +class JsonLogAttributes : public beast::Journal::StructuredLogAttributes +{ +public: + using AttributeFields = std::unordered_map; + using Pair = AttributeFields::value_type; + + explicit JsonLogAttributes(AttributeFields contextValues = {}); + + void + setModuleName(std::string const& name) override; + + std::unique_ptr + clone() const override; + + void + combine(std::unique_ptr const& context) override; + + void + combine(std::unique_ptr&& context) override; + + AttributeFields& + contextValues() + { + return contextValues_; + } + +private: + AttributeFields contextValues_; +}; + +class JsonStructuredJournal : public beast::Journal::StructuredJournalImpl +{ +private: + struct Logger + { + std::source_location location = {}; + Json::Value messageParams; + + Logger() = default; + Logger( + JsonStructuredJournal const* journal, + std::source_location location); + + void + write( + beast::Journal::Sink* sink, + beast::severities::Severity level, + std::string const& text, + beast::Journal::StructuredLogAttributes* context) const; + }; + + [[nodiscard]] Logger + logger(std::source_location location) const; + + static thread_local Logger currentLogger_; + + template + friend std::ostream& + operator<<(std::ostream& os, LogParameter const&); + + template + friend std::ostream& + operator<<(std::ostream& os, LogField const&); + +public: + void + initMessageContext(std::source_location location) override; + + void + flush( + beast::Journal::Sink* sink, + beast::severities::Severity level, + std::string const& text, + beast::Journal::StructuredLogAttributes* context) override; +}; + +template +std::ostream& +operator<<(std::ostream& os, LogParameter const& param) +{ + using ValueType = std::decay_t; + // TODO: Update the Json library to support 64-bit integer values. + if constexpr ( + std::constructible_from && + (!std::is_integral_v || + sizeof(ValueType) <= sizeof(Json::Int))) + { + JsonStructuredJournal::currentLogger_.messageParams[param.name_] = + Json::Value{param.value_}; + return os << param.value_; + } + else + { + std::ostringstream oss; + oss << param.value_; + + JsonStructuredJournal::currentLogger_.messageParams[param.name_] = + oss.str(); + return os << oss.str(); + } +} + +template +std::ostream& +operator<<(std::ostream& os, LogField const& param) +{ + using ValueType = std::decay_t; + // TODO: Update the Json library to support 64-bit integer values. + if constexpr ( + std::constructible_from && + (!std::is_integral_v || + sizeof(ValueType) <= sizeof(Json::Int))) + { + JsonStructuredJournal::currentLogger_.messageParams[param.name_] = + Json::Value{param.value_}; + } + else + { + std::ostringstream oss; + oss << param.value_; + + JsonStructuredJournal::currentLogger_.messageParams[param.name_] = + oss.str(); + } + return os; +} + +template +LogParameter +param(char const* name, T&& value) +{ + return LogParameter{name, std::forward(value)}; +} + +template +LogField +field(char const* name, T&& value) +{ + return LogField{name, std::forward(value)}; +} + +[[nodiscard]] inline std::unique_ptr +attributes(std::initializer_list const& fields) +{ + return std::make_unique(fields); +} + +} // namespace log +} // namespace ripple + +#endif diff --git a/include/xrpl/resource/detail/Logic.h b/include/xrpl/resource/detail/Logic.h index b07ee00e73a..b8288294fe9 100644 --- a/include/xrpl/resource/detail/Logic.h +++ b/include/xrpl/resource/detail/Logic.h @@ -32,6 +32,7 @@ #include #include #include +#include #include @@ -132,7 +133,8 @@ class Logic } } - JLOG(m_journal.debug()) << "New inbound endpoint " << *entry; + JLOG(m_journal.debug()) + << "New inbound endpoint " << log::param("Entry", *entry); return Consumer(*this, *entry); } @@ -160,7 +162,8 @@ class Logic } } - JLOG(m_journal.debug()) << "New outbound endpoint " << *entry; + JLOG(m_journal.debug()) + << "New outbound endpoint " << log::param("Entry", *entry); return Consumer(*this, *entry); } @@ -193,7 +196,8 @@ class Logic } } - JLOG(m_journal.debug()) << "New unlimited endpoint " << *entry; + JLOG(m_journal.debug()) + << "New unlimited endpoint " << log::param("Entry", *entry); return Consumer(*this, *entry); } @@ -350,7 +354,8 @@ class Logic { if (iter->whenExpires <= elapsed) { - JLOG(m_journal.debug()) << "Expired " << *iter; + JLOG(m_journal.debug()) + << "Expired " << log::param("Entry", *iter); auto table_iter = table_.find(*iter->key); ++iter; erase(table_iter); @@ -422,7 +427,9 @@ class Logic std::lock_guard _(lock_); if (--entry.refcount == 0) { - JLOG(m_journal.debug()) << "Inactive " << entry; + JLOG(m_journal.debug()) + << "Inactive " << log::param("Entry", entry); + ; switch (entry.key->kind) { @@ -474,7 +481,8 @@ class Logic clock_type::time_point const now(m_clock.now()); int const balance(entry.add(fee.cost(), now)); JLOG(getStream(fee.cost(), m_journal)) - << "Charging " << entry << " for " << fee << context; + << "Charging " << log::param("Entry", entry) << " for " + << log::param("Fee", fee) << context; return disposition(balance); } @@ -496,7 +504,9 @@ class Logic } if (notify) { - JLOG(m_journal.info()) << "Load warning: " << entry; + JLOG(m_journal.info()) + << "Load warning: " << log::param("Entry", entry); + ; ++m_stats.warn; } return notify; @@ -515,8 +525,10 @@ class Logic if (balance >= dropThreshold) { JLOG(m_journal.warn()) - << "Consumer entry " << entry << " dropped with balance " - << balance << " at or above drop threshold " << dropThreshold; + << "Consumer entry " << log::param("Entry", entry) + << " dropped with balance " << log::param("Entry", balance) + << " at or above drop threshold " + << log::param("Entry", dropThreshold); // Adding feeDrop at this point keeps the dropped connection // from re-connecting for at least a little while after it is diff --git a/include/xrpl/server/detail/BasePeer.h b/include/xrpl/server/detail/BasePeer.h index 30de63e6ff0..bdae5046198 100644 --- a/include/xrpl/server/detail/BasePeer.h +++ b/include/xrpl/server/detail/BasePeer.h @@ -25,6 +25,7 @@ #include #include #include +#include #include @@ -47,7 +48,6 @@ class BasePeer : public io_list::work Port const& port_; Handler& handler_; endpoint_type remote_address_; - beast::WrappedSink sink_; beast::Journal const j_; boost::asio::executor_work_guard work_; @@ -84,13 +84,13 @@ BasePeer::BasePeer( : port_(port) , handler_(handler) , remote_address_(remote_address) - , sink_( - journal.sink(), - [] { - static std::atomic id{0}; - return "##" + std::to_string(++id) + " "; - }()) - , j_(sink_) + , j_(journal, + log::attributes( + {{"PeerID", + [] { + static std::atomic id{0}; + return "##" + std::to_string(++id) + " "; + }()}})) , work_(boost::asio::make_work_guard(executor)) , strand_(boost::asio::make_strand(executor)) { diff --git a/include/xrpl/telemetry/JsonLogs.h b/include/xrpl/telemetry/JsonLogs.h new file mode 100644 index 00000000000..df8fe47625a --- /dev/null +++ b/include/xrpl/telemetry/JsonLogs.h @@ -0,0 +1,221 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2025 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_LOGGING_STRUCTUREDJOURNAL_H_INCLUDED +#define RIPPLE_LOGGING_STRUCTUREDJOURNAL_H_INCLUDED + +#include +#include + +#include +#include +#include + +namespace ripple { +namespace log { + +template +class LogParameter +{ +public: + template + LogParameter(char const* name, TArg&& value) + : name_(name), value_(std::forward(value)) + { + } + +private: + char const* name_; + T value_; + + template + friend std::ostream& + operator<<(std::ostream& os, LogParameter const&); +}; + +template +class LogField +{ +public: + template + LogField(char const* name, TArg&& value) + : name_(name), value_(std::forward(value)) + { + } + +private: + char const* name_; + T value_; + + template + friend std::ostream& + operator<<(std::ostream& os, LogField const&); +}; + +class JsonLogAttributes : public beast::Journal::StructuredLogAttributes +{ +public: + using AttributeFields = std::unordered_map; + using Pair = AttributeFields::value_type; + + explicit JsonLogAttributes(AttributeFields contextValues = {}); + + void + setModuleName(std::string const& name) override; + + std::unique_ptr + clone() const override; + + void + combine(std::unique_ptr const& context) override; + + void + combine(std::unique_ptr&& context) override; + + AttributeFields& + contextValues() + { + return contextValues_; + } + +private: + AttributeFields contextValues_; +}; + +class JsonStructuredJournal : public beast::Journal::StructuredJournalImpl +{ +private: + struct Logger + { + std::source_location location = {}; + Json::Value messageParams; + + Logger() = default; + Logger( + JsonStructuredJournal const* journal, + std::source_location location); + + void + write( + beast::Journal::Sink* sink, + beast::severities::Severity level, + std::string const& text, + beast::Journal::StructuredLogAttributes* context) const; + }; + + [[nodiscard]] Logger + logger(std::source_location location) const; + + static thread_local Logger currentLogger_; + + template + friend std::ostream& + operator<<(std::ostream& os, LogParameter const&); + + template + friend std::ostream& + operator<<(std::ostream& os, LogField const&); + +public: + void + initMessageContext(std::source_location location) override; + + void + flush( + beast::Journal::Sink* sink, + beast::severities::Severity level, + std::string const& text, + beast::Journal::StructuredLogAttributes* context) override; +}; + +template +std::ostream& +operator<<(std::ostream& os, LogParameter const& param) +{ + using ValueType = std::decay_t; + // TODO: Update the Json library to support 64-bit integer values. + if constexpr ( + std::constructible_from && + (!std::is_integral_v || + sizeof(ValueType) <= sizeof(Json::Int))) + { + JsonStructuredJournal::currentLogger_.messageParams[param.name_] = + Json::Value{param.value_}; + return os << param.value_; + } + else + { + std::ostringstream oss; + oss << param.value_; + + JsonStructuredJournal::currentLogger_.messageParams[param.name_] = + oss.str(); + return os << oss.str(); + } +} + +template +std::ostream& +operator<<(std::ostream& os, LogField const& param) +{ + using ValueType = std::decay_t; + // TODO: Update the Json library to support 64-bit integer values. + if constexpr ( + std::constructible_from && + (!std::is_integral_v || + sizeof(ValueType) <= sizeof(Json::Int))) + { + JsonStructuredJournal::currentLogger_.messageParams[param.name_] = + Json::Value{param.value_}; + } + else + { + std::ostringstream oss; + oss << param.value_; + + JsonStructuredJournal::currentLogger_.messageParams[param.name_] = + oss.str(); + } + return os; +} + +template +LogParameter +param(char const* name, T&& value) +{ + return LogParameter{name, std::forward(value)}; +} + +template +LogField +field(char const* name, T&& value) +{ + return LogField{name, std::forward(value)}; +} + +[[nodiscard]] inline std::unique_ptr +attributes(std::initializer_list const& fields) +{ + return std::make_unique(fields); +} + +} // namespace log +} // namespace ripple + +#endif diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index 14873a3fd79..eba549d078a 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -38,6 +38,9 @@ namespace ripple { +std::unique_ptr + Logs::globalLogAttributes_; + Logs::Sink::Sink( std::string const& partition, beast::severities::Severity thresh, @@ -157,9 +160,22 @@ Logs::operator[](std::string const& name) } beast::Journal -Logs::journal(std::string const& name) +Logs::journal( + std::string const& name, + std::unique_ptr attributes) { - return beast::Journal(get(name)); + if (globalLogAttributes_) + { + if (attributes) + { + attributes->combine(globalLogAttributes_); + } + else + { + attributes = globalLogAttributes_->clone(); + } + } + return beast::Journal(get(name), name, std::move(attributes)); } beast::severities::Severity @@ -332,36 +348,39 @@ Logs::format( { output.reserve(message.size() + partition.size() + 100); - output = to_string(std::chrono::system_clock::now()); + if (!beast::Journal::isStructuredJournalEnabled()) + { + output = to_string(std::chrono::system_clock::now()); - output += " "; - if (!partition.empty()) - output += partition + ":"; + output += " "; + if (!partition.empty()) + output += partition + ":"; - using namespace beast::severities; - switch (severity) - { - case kTrace: - output += "TRC "; - break; - case kDebug: - output += "DBG "; - break; - case kInfo: - output += "NFO "; - break; - case kWarning: - output += "WRN "; - break; - case kError: - output += "ERR "; - break; - default: - UNREACHABLE("ripple::Logs::format : invalid severity"); - [[fallthrough]]; - case kFatal: - output += "FTL "; - break; + using namespace beast::severities; + switch (severity) + { + case kTrace: + output += "TRC "; + break; + case kDebug: + output += "DBG "; + break; + case kInfo: + output += "NFO "; + break; + case kWarning: + output += "WRN "; + break; + case kError: + output += "ERR "; + break; + default: + UNREACHABLE("ripple::Logs::format : invalid severity"); + [[fallthrough]]; + case kFatal: + output += "FTL "; + break; + } } output += message; diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index 828f2fa619e..de9a5f196fc 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -25,6 +25,8 @@ namespace beast { +Journal::StructuredJournalImpl* Journal::m_structuredJournalImpl = nullptr; + //------------------------------------------------------------------------------ // A Sink that does nothing. @@ -87,6 +89,29 @@ Journal::getNullSink() //------------------------------------------------------------------------------ +std::string +severities::to_string(Severity severity) +{ + switch (severity) + { + case kDisabled: + return "disabled"; + case kTrace: + return "trace"; + case kDebug: + return "debug"; + case kInfo: + return "info"; + case kWarning: + return "warning"; + case kError: + return "error"; + case kFatal: + return "fatal"; + default: + assert(false); + } +} Journal::Sink::Sink(Severity thresh, bool console) : thresh_(thresh), m_console(console) { @@ -126,17 +151,21 @@ Journal::Sink::threshold(Severity thresh) //------------------------------------------------------------------------------ -Journal::ScopedStream::ScopedStream(Sink& sink, Severity level) - : m_sink(sink), m_level(level) +Journal::ScopedStream::ScopedStream( + std::unique_ptr attributes, + Sink& sink, + Severity level) + : m_attributes(std::move(attributes)), m_sink(sink), m_level(level) { // Modifiers applied from all ctors m_ostream << std::boolalpha << std::showbase; } Journal::ScopedStream::ScopedStream( + std::unique_ptr attributes, Stream const& stream, std::ostream& manip(std::ostream&)) - : ScopedStream(stream.sink(), stream.level()) + : ScopedStream(std::move(attributes), stream.sink(), stream.level()) { m_ostream << manip; } @@ -147,9 +176,29 @@ Journal::ScopedStream::~ScopedStream() if (!s.empty()) { if (s == "\n") - m_sink.write(m_level, ""); + { + if (m_structuredJournalImpl) + { + m_structuredJournalImpl->flush( + &m_sink, m_level, "", m_attributes.get()); + } + else + { + m_sink.write(m_level, ""); + } + } else - m_sink.write(m_level, s); + { + if (m_structuredJournalImpl) + { + m_structuredJournalImpl->flush( + &m_sink, m_level, s, m_attributes.get()); + } + else + { + m_sink.write(m_level, s); + } + } } } @@ -164,7 +213,7 @@ Journal::ScopedStream::operator<<(std::ostream& manip(std::ostream&)) const Journal::ScopedStream Journal::Stream::operator<<(std::ostream& manip(std::ostream&)) const { - return ScopedStream(*this, manip); + return {m_attributes ? m_attributes->clone() : nullptr, *this, manip}; } } // namespace beast diff --git a/src/libxrpl/logging/JsonLogs.cpp b/src/libxrpl/logging/JsonLogs.cpp new file mode 100644 index 00000000000..869b6ed09e7 --- /dev/null +++ b/src/libxrpl/logging/JsonLogs.cpp @@ -0,0 +1,136 @@ +//------------------------------------------------------------------------------ +/* + This file is part of Beast: https://github.com/vinniefalco/Beast + Copyright 2013, Vinnie Falco + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +namespace ripple { +namespace log { + +thread_local JsonStructuredJournal::Logger + JsonStructuredJournal::currentLogger_{}; + +JsonLogAttributes::JsonLogAttributes(AttributeFields contextValues) + : contextValues_(std::move(contextValues)) +{ +} + +void +JsonLogAttributes::setModuleName(std::string const& name) +{ + contextValues()["Module"] = name; +} + +std::unique_ptr +JsonLogAttributes::clone() const +{ + return std::make_unique(*this); +} + +void +JsonLogAttributes::combine( + std::unique_ptr const& context) +{ + auto structuredContext = + static_cast(context.get()); + contextValues_.insert( + structuredContext->contextValues_.begin(), + structuredContext->contextValues_.end()); +} + +void +JsonLogAttributes::combine(std::unique_ptr&& context) +{ + auto structuredContext = + static_cast(context.get()); + + if (contextValues_.empty()) + { + contextValues_ = std::move(structuredContext->contextValues_); + } + else + { + contextValues_.insert( + structuredContext->contextValues_.begin(), + structuredContext->contextValues_.end()); + } +} + +JsonStructuredJournal::Logger::Logger( + JsonStructuredJournal const* journal, + std::source_location location) + : location(location) +{ +} + +void +JsonStructuredJournal::Logger::write( + beast::Journal::Sink* sink, + beast::severities::Severity level, + std::string const& text, + beast::Journal::StructuredLogAttributes* context) const +{ + Json::Value globalContext; + if (context) + { + auto jsonContext = static_cast(context); + for (auto const& [key, value] : jsonContext->contextValues()) + { + globalContext[key] = value; + } + } + globalContext["Function"] = location.function_name(); + globalContext["File"] = location.file_name(); + globalContext["Line"] = location.line(); + std::stringstream threadIdStream; + threadIdStream << std::this_thread::get_id(); + globalContext["ThreadId"] = threadIdStream.str(); + globalContext["Params"] = messageParams; + globalContext["Level"] = Logs::toString(Logs::fromSeverity(level)); + globalContext["Message"] = text; + globalContext["Time"] = + to_string(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count()); + + sink->write(level, Json::jsonAsString(globalContext)); +} + +JsonStructuredJournal::Logger +JsonStructuredJournal::logger(std::source_location location) const +{ + return Logger{this, location}; +} + +void +JsonStructuredJournal::initMessageContext(std::source_location location) +{ + currentLogger_ = logger(location); +} + +void +JsonStructuredJournal::flush( + beast::Journal::Sink* sink, + beast::severities::Severity level, + std::string const& text, + beast::Journal::StructuredLogAttributes* context) +{ + currentLogger_.write(sink, level, text, context); +} + +} // namespace log +} // namespace ripple diff --git a/src/libxrpl/telemetry/JsonLogs.cpp b/src/libxrpl/telemetry/JsonLogs.cpp new file mode 100644 index 00000000000..9c9b8762d8e --- /dev/null +++ b/src/libxrpl/telemetry/JsonLogs.cpp @@ -0,0 +1,131 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2025 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include + +namespace ripple::log { + +thread_local JsonStructuredJournal::Logger + JsonStructuredJournal::currentLogger_{}; + +JsonLogAttributes::JsonLogAttributes(AttributeFields contextValues) + : contextValues_(std::move(contextValues)) +{ +} + +void +JsonLogAttributes::setModuleName(std::string const& name) +{ + contextValues()["Module"] = name; +} + +std::unique_ptr +JsonLogAttributes::clone() const +{ + return std::make_unique(*this); +} + +void +JsonLogAttributes::combine( + std::unique_ptr const& context) +{ + auto structuredContext = + static_cast(context.get()); + contextValues_.merge(AttributeFields{structuredContext->contextValues_}); +} + +void +JsonLogAttributes::combine(std::unique_ptr&& context) +{ + auto structuredContext = static_cast(context.get()); + + if (contextValues_.empty()) + { + contextValues_ = std::move(structuredContext->contextValues_); + } + else + { + contextValues_.merge(structuredContext->contextValues_); + } +} + +JsonStructuredJournal::Logger::Logger( + JsonStructuredJournal const* journal, + std::source_location location) + : location(location) +{ +} + +void +JsonStructuredJournal::Logger::write( + beast::Journal::Sink* sink, + beast::severities::Severity level, + std::string const& text, + beast::Journal::StructuredLogAttributes* context) const +{ + Json::Value globalContext; + if (context) + { + auto jsonContext = static_cast(context); + for (auto const& [key, value] : jsonContext->contextValues()) + { + globalContext[key] = value; + } + } + globalContext["Function"] = location.function_name(); + globalContext["File"] = location.file_name(); + globalContext["Line"] = location.line(); + std::stringstream threadIdStream; + threadIdStream << std::this_thread::get_id(); + globalContext["ThreadId"] = threadIdStream.str(); + globalContext["Params"] = messageParams; + globalContext["Level"] = to_string(level); + globalContext["Message"] = text; + globalContext["Time"] = + to_string(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count()); + + sink->write(level, to_string(globalContext)); +} + +JsonStructuredJournal::Logger +JsonStructuredJournal::logger(std::source_location location) const +{ + return Logger{this, location}; +} + +void +JsonStructuredJournal::initMessageContext(std::source_location location) +{ + currentLogger_ = logger(location); +} + +void +JsonStructuredJournal::flush( + beast::Journal::Sink* sink, + beast::severities::Severity level, + std::string const& text, + beast::Journal::StructuredLogAttributes* context) +{ + currentLogger_.write(sink, level, text, context); +} + +} // namespace ripple::log diff --git a/src/test/csf/Peer.h b/src/test/csf/Peer.h index 1cb2d03cc6d..783da4f2494 100644 --- a/src/test/csf/Peer.h +++ b/src/test/csf/Peer.h @@ -33,6 +33,7 @@ #include #include +#include #include #include @@ -178,7 +179,6 @@ struct Peer using NodeKey = Validation::NodeKey; //! Logging support that prefixes messages with the peer ID - beast::WrappedSink sink; beast::Journal j; //! Generic consensus @@ -284,8 +284,7 @@ struct Peer TrustGraph& tg, CollectorRefs& c, beast::Journal jIn) - : sink(jIn, "Peer " + to_string(i) + ": ") - , j(sink) + : j(jIn, log::attributes({{"Peer", "Peer " + to_string(i)}})) , consensus(s.clock(), *this, j) , id{i} , key{id, 0} diff --git a/src/tests/libxrpl/CMakeLists.txt b/src/tests/libxrpl/CMakeLists.txt index f97283c9550..73520a51282 100644 --- a/src/tests/libxrpl/CMakeLists.txt +++ b/src/tests/libxrpl/CMakeLists.txt @@ -14,3 +14,5 @@ xrpl_add_test(crypto) target_link_libraries(xrpl.test.crypto PRIVATE xrpl.imports.test) xrpl_add_test(net) target_link_libraries(xrpl.test.net PRIVATE xrpl.imports.test) +xrpl_add_test(telemetry) +target_link_libraries(xrpl.test.telemetry PRIVATE xrpl.imports.test) diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp new file mode 100644 index 00000000000..8ce69af68d9 --- /dev/null +++ b/src/tests/libxrpl/basics/log.cpp @@ -0,0 +1,169 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include + +#include + +using namespace ripple; + +class MockLogs : public Logs +{ +private: + class Sink : public beast::Journal::Sink + { + private: + MockLogs& logs_; + std::string partition_; + + public: + Sink( + std::string const& partition, + beast::severities::Severity thresh, + MockLogs& logs) + : beast::Journal::Sink(thresh, false) + , logs_(logs) + , partition_(partition) + { + } + + Sink(Sink const&) = delete; + Sink& + operator=(Sink const&) = delete; + + void + write(beast::severities::Severity level, std::string const& text) + override + { + logs_.logStream_ << text; + } + + void + writeAlways(beast::severities::Severity level, std::string const& text) + override + { + logs_.logStream_ << text; + } + }; + + std::stringstream& logStream_; + +public: + MockLogs(std::stringstream& logStream, beast::severities::Severity level) + : Logs(level), logStream_(logStream) + { + } + + virtual std::unique_ptr + makeSink( + std::string const& partition, + beast::severities::Severity startingLevel) + { + return std::make_unique(partition, startingLevel, *this); + } +}; + +TEST_CASE("Enable Json Logs") +{ + static log::JsonStructuredJournal structuredJournal; + + std::stringstream logStream; + + MockLogs logs{logStream, beast::severities::kAll}; + + logs.journal("Test").debug() << "Test"; + + CHECK(logStream.str() == "Test"); + + logStream.str(""); + + beast::Journal::enableStructuredJournal(&structuredJournal); + + logs.journal("Test").debug() << "Test"; + + Json::Reader reader; + Json::Value jsonLog; + bool result = reader.parse(logStream.str(), jsonLog); + + CHECK(result); + + CHECK(jsonLog.isObject()); + CHECK(jsonLog.isMember("Message")); + CHECK(jsonLog["Message"].isString()); + CHECK(jsonLog["Message"].asString() == "Test"); +} + +TEST_CASE("Global attributes") +{ + static log::JsonStructuredJournal structuredJournal; + + std::stringstream logStream; + + MockLogs logs{logStream, beast::severities::kAll}; + + beast::Journal::enableStructuredJournal(&structuredJournal); + MockLogs::setGlobalAttributes(log::attributes({{"Field1", "Value1"}})); + + logs.journal("Test").debug() << "Test"; + + Json::Reader reader; + Json::Value jsonLog; + bool result = reader.parse(logStream.str(), jsonLog); + + CHECK(result); + + CHECK(jsonLog.isObject()); + CHECK(jsonLog.isMember("Field1")); + CHECK(jsonLog["Field1"].isString()); + CHECK(jsonLog["Field1"].asString() == "Value1"); +} + +TEST_CASE("Global attributes inheritable") +{ + static log::JsonStructuredJournal structuredJournal; + + std::stringstream logStream; + + MockLogs logs{logStream, beast::severities::kAll}; + + beast::Journal::enableStructuredJournal(&structuredJournal); + MockLogs::setGlobalAttributes(log::attributes({{"Field1", "Value1"}})); + + logs.journal( + "Test", + log::attributes({{"Field1", "Value3"}, {"Field2", "Value2"}})) + .debug() + << "Test"; + + Json::Reader reader; + Json::Value jsonLog; + bool result = reader.parse(logStream.str(), jsonLog); + + CHECK(result); + + CHECK(jsonLog.isObject()); + CHECK(jsonLog.isMember("Field1")); + CHECK(jsonLog["Field1"].isString()); + // Field1 should be overwritten to Value3 + CHECK(jsonLog["Field1"].asString() == "Value3"); + CHECK(jsonLog["Field2"].isString()); + CHECK(jsonLog["Field2"].asString() == "Value2"); +} \ No newline at end of file diff --git a/src/tests/libxrpl/telemetry/base64.cpp b/src/tests/libxrpl/telemetry/base64.cpp new file mode 100644 index 00000000000..fe9b86abb10 --- /dev/null +++ b/src/tests/libxrpl/telemetry/base64.cpp @@ -0,0 +1,67 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include + +#include + +#include + +using namespace ripple; + +static void +check(std::string const& in, std::string const& out) +{ + auto const encoded = base64_encode(in); + CHECK(encoded == out); + CHECK(base64_decode(encoded) == in); +} + +TEST_CASE("base64") +{ + check("", ""); + check("f", "Zg=="); + check("fo", "Zm8="); + check("foo", "Zm9v"); + check("foob", "Zm9vYg=="); + check("fooba", "Zm9vYmE="); + check("foobar", "Zm9vYmFy"); + + check( + "Man is distinguished, not only by his reason, but by this " + "singular passion from " + "other animals, which is a lust of the mind, that by a " + "perseverance of delight " + "in the continued and indefatigable generation of knowledge, " + "exceeds the short " + "vehemence of any carnal pleasure.", + "TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dC" + "BieSB0aGlz" + "IHNpbmd1bGFyIHBhc3Npb24gZnJvbSBvdGhlciBhbmltYWxzLCB3aGljaCBpcyBhIG" + "x1c3Qgb2Yg" + "dGhlIG1pbmQsIHRoYXQgYnkgYSBwZXJzZXZlcmFuY2Ugb2YgZGVsaWdodCBpbiB0aG" + "UgY29udGlu" + "dWVkIGFuZCBpbmRlZmF0aWdhYmxlIGdlbmVyYXRpb24gb2Yga25vd2xlZGdlLCBleG" + "NlZWRzIHRo" + "ZSBzaG9ydCB2ZWhlbWVuY2Ugb2YgYW55IGNhcm5hbCBwbGVhc3VyZS4="); + + std::string const notBase64 = "not_base64!!"; + std::string const truncated = "not"; + CHECK(base64_decode(notBase64) == base64_decode(truncated)); +} diff --git a/src/tests/libxrpl/telemetry/json_logs.cpp b/src/tests/libxrpl/telemetry/json_logs.cpp new file mode 100644 index 00000000000..209606dc8ca --- /dev/null +++ b/src/tests/libxrpl/telemetry/json_logs.cpp @@ -0,0 +1,353 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include + +#include + +using namespace ripple; + +/** + * @brief sink for writing all log messages to a stringstream + */ +class MockSink : public beast::Journal::Sink +{ + std::stringstream& strm_; + +public: + MockSink(beast::severities::Severity threshold, std::stringstream& strm) + : beast::Journal::Sink(threshold, false), strm_(strm) + { + } + + void + write(beast::severities::Severity level, std::string const& text) override + { + strm_ << text; + } + + void + writeAlways(beast::severities::Severity level, std::string const& text) + override + { + strm_ << text; + } +}; + +class JsonLogStreamFixture +{ +public: + JsonLogStreamFixture() + : sink_(beast::severities::kAll, logStream_), j_(sink_) + { + static log::JsonStructuredJournal structuredJournal; + beast::Journal::enableStructuredJournal(&structuredJournal); + } + + std::stringstream& + stream() + { + return logStream_; + } + + beast::Journal& + journal() + { + return j_; + } + +private: + MockSink sink_; + std::stringstream logStream_; + beast::Journal j_; +}; + +TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") +{ + journal().debug() << "Test"; + + Json::Value logValue; + Json::Reader reader; + reader.parse(stream().str(), logValue); + + CHECK(logValue.isObject()); + CHECK(logValue.isMember("Function")); + CHECK(logValue.isMember("File")); + CHECK(logValue.isMember("Line")); + CHECK(logValue.isMember("ThreadId")); + CHECK(logValue.isMember("Params")); + CHECK(logValue.isMember("Level")); + CHECK(logValue.isMember("Message")); + CHECK(logValue.isMember("Time")); + + CHECK(logValue["Function"].isString()); + CHECK(logValue["File"].isString()); + CHECK(logValue["Line"].isNumeric()); + CHECK(logValue["Params"].isNull()); + CHECK(logValue["Message"].isString()); + CHECK(logValue["Message"].asString() == "Test"); +} + +TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") +{ + { + stream().str(""); + journal().trace() << "Test"; + Json::Value logValue; + Json::Reader reader; + reader.parse(stream().str(), logValue); + + CHECK( + logValue["Level"].asString() == + beast::severities::to_string(beast::severities::kTrace)); + } + + { + stream().str(""); + journal().debug() << "Test"; + Json::Value logValue; + Json::Reader reader; + reader.parse(stream().str(), logValue); + + CHECK( + logValue["Level"].asString() == + beast::severities::to_string(beast::severities::kDebug)); + } + + { + stream().str(""); + journal().info() << "Test"; + Json::Value logValue; + Json::Reader reader; + reader.parse(stream().str(), logValue); + + CHECK( + logValue["Level"].asString() == + beast::severities::to_string(beast::severities::kInfo)); + } + + { + stream().str(""); + journal().warn() << "Test"; + Json::Value logValue; + Json::Reader reader; + reader.parse(stream().str(), logValue); + + CHECK( + logValue["Level"].asString() == + beast::severities::to_string(beast::severities::kWarning)); + } + + { + stream().str(""); + journal().error() << "Test"; + Json::Value logValue; + Json::Reader reader; + reader.parse(stream().str(), logValue); + + CHECK( + logValue["Level"].asString() == + beast::severities::to_string(beast::severities::kError)); + } + + { + stream().str(""); + journal().fatal() << "Test"; + Json::Value logValue; + Json::Reader reader; + reader.parse(stream().str(), logValue); + + CHECK( + logValue["Level"].asString() == + beast::severities::to_string(beast::severities::kFatal)); + } +} + +TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogStream") +{ + journal().stream(beast::severities::kError) << "Test"; + + Json::Value logValue; + Json::Reader reader; + reader.parse(stream().str(), logValue); + + CHECK( + logValue["Level"].asString() == + beast::severities::to_string(beast::severities::kError)); +} + +TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogParams") +{ + journal().debug() << "Test: " << log::param("Field1", 1) << ", " + << log::param( + "Field2", + std::numeric_limits::max()); + + Json::Value logValue; + Json::Reader reader; + reader.parse(stream().str(), logValue); + + CHECK(logValue["Params"].isObject()); + CHECK(logValue["Params"]["Field1"].isNumeric()); + CHECK(logValue["Params"]["Field1"].asInt() == 1); + // UInt64 doesn't fit in Json::Value so it should be converted to a string + // NOTE: We should expect it to be an int64 after we make the json library + // support in64 and uint64 + CHECK(logValue["Params"]["Field2"].isString()); + CHECK(logValue["Params"]["Field2"].asString() == "18446744073709551615"); + CHECK(logValue["Message"].isString()); + CHECK(logValue["Message"].asString() == "Test: 1, 18446744073709551615"); +} + +TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") +{ + journal().debug() << "Test" << log::field("Field1", 1) + << log::field( + "Field2", + std::numeric_limits::max()); + + Json::Value logValue; + Json::Reader reader; + reader.parse(stream().str(), logValue); + + CHECK(logValue["Params"].isObject()); + CHECK(logValue["Params"]["Field1"].isNumeric()); + CHECK(logValue["Params"]["Field1"].asInt() == 1); + // UInt64 doesn't fit in Json::Value so it should be converted to a string + // NOTE: We should expect it to be an int64 after we make the json library + // support in64 and uint64 + CHECK(logValue["Params"]["Field2"].isString()); + CHECK(logValue["Params"]["Field2"].asString() == "18446744073709551615"); + CHECK(logValue["Message"].isString()); + CHECK(logValue["Message"].asString() == "Test"); +} + +TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributes") +{ + beast::Journal j{ + journal(), log::attributes({{"Field1", "Value1"}, {"Field2", 2}})}; + + j.debug() << "Test"; + + Json::Value logValue; + Json::Reader reader; + reader.parse(stream().str(), logValue); + + CHECK(logValue["Field1"].isString()); + CHECK(logValue["Field1"].asString() == "Value1"); + CHECK(logValue["Field2"].isNumeric()); + CHECK(logValue["Field2"].asInt() == 2); +} + +TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributesInheritable") +{ + beast::Journal j{ + journal(), log::attributes({{"Field1", "Value1"}, {"Field2", 2}})}; + beast::Journal j2{ + j, log::attributes({{"Field3", "Value3"}, {"Field2", 0}})}; + + j2.debug() << "Test"; + + Json::Value logValue; + Json::Reader reader; + reader.parse(stream().str(), logValue); + + CHECK(logValue["Field1"].isString()); + CHECK(logValue["Field1"].asString() == "Value1"); + CHECK(logValue["Field3"].isString()); + CHECK(logValue["Field3"].asString() == "Value3"); + // Field2 should be overwritten to 0 + CHECK(logValue["Field2"].isNumeric()); + CHECK(logValue["Field2"].asInt() == 0); +} + +TEST_CASE_FIXTURE( + JsonLogStreamFixture, + "TestJournalAttributesInheritableAfterMoving") +{ + beast::Journal j{ + std::move(journal()), + log::attributes({{"Field1", "Value1"}, {"Field2", 2}})}; + beast::Journal j2{ + std::move(j), log::attributes({{"Field3", "Value3"}, {"Field2", 0}})}; + + j2.debug() << "Test"; + + Json::Value logValue; + Json::Reader reader; + reader.parse(stream().str(), logValue); + + CHECK(logValue["Field1"].isString()); + CHECK(logValue["Field1"].asString() == "Value1"); + CHECK(logValue["Field3"].isString()); + CHECK(logValue["Field3"].asString() == "Value3"); + // Field2 should be overwritten to 0 + CHECK(logValue["Field2"].isNumeric()); + CHECK(logValue["Field2"].asInt() == 0); +} + +TEST_CASE_FIXTURE( + JsonLogStreamFixture, + "TestJournalAttributesInheritableAfterCopyAssignment") +{ + beast::Journal j{ + std::move(journal()), + log::attributes({{"Field1", "Value1"}, {"Field2", 2}})}; + + beast::Journal j2{beast::Journal::getNullSink()}; + + j2 = j; + + j2.debug() << "Test"; + + Json::Value logValue; + Json::Reader reader; + reader.parse(stream().str(), logValue); + + CHECK(logValue["Field1"].isString()); + CHECK(logValue["Field1"].asString() == "Value1"); + CHECK(logValue["Field2"].isNumeric()); + CHECK(logValue["Field2"].asInt() == 2); +} + +TEST_CASE_FIXTURE( + JsonLogStreamFixture, + "TestJournalAttributesInheritableAfterMoveAssignment") +{ + beast::Journal j{ + std::move(journal()), + log::attributes({{"Field1", "Value1"}, {"Field2", 2}})}; + + beast::Journal j2{beast::Journal::getNullSink()}; + + j2 = std::move(j); + + j2.debug() << "Test"; + + Json::Value logValue; + Json::Reader reader; + reader.parse(stream().str(), logValue); + + CHECK(logValue["Field1"].isString()); + CHECK(logValue["Field1"].asString() == "Value1"); + CHECK(logValue["Field2"].isNumeric()); + CHECK(logValue["Field2"].asInt() == 2); +} \ No newline at end of file diff --git a/src/tests/libxrpl/telemetry/main.cpp b/src/tests/libxrpl/telemetry/main.cpp new file mode 100644 index 00000000000..0a3f254ea87 --- /dev/null +++ b/src/tests/libxrpl/telemetry/main.cpp @@ -0,0 +1,2 @@ +#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN +#include diff --git a/src/xrpld/app/consensus/RCLConsensus.cpp b/src/xrpld/app/consensus/RCLConsensus.cpp index 292ba7d483d..47f6cc70f61 100644 --- a/src/xrpld/app/consensus/RCLConsensus.cpp +++ b/src/xrpld/app/consensus/RCLConsensus.cpp @@ -1107,8 +1107,10 @@ RCLConsensus::startRound( RclConsensusLogger::RclConsensusLogger( char const* label, bool const validating, - beast::Journal j) - : j_(j) + beast::Journal j, + std::source_location location) + : j_(j, log::attributes({{"Role", "ConsensusLogger"}, {"Label", label}})) + , location_(location) { if (!validating && !j.info()) return; @@ -1125,11 +1127,11 @@ RclConsensusLogger::~RclConsensusLogger() return; auto const duration = std::chrono::duration_cast( std::chrono::steady_clock::now() - start_); - std::stringstream outSs; - outSs << header_ << "duration " << (duration.count() / 1000) << '.' - << std::setw(3) << std::setfill('0') << (duration.count() % 1000) - << "s. " << ss_->str(); - j_.sink().writeAlways(beast::severities::kInfo, outSs.str()); + + j_.info(location_) << header_ << "duration " << (duration.count() / 1000) + << '.' << std::setw(3) << std::setfill('0') + << (duration.count() % 1000) << "s. " << ss_->str() + << log::field("Duration", duration.count()); } } // namespace ripple diff --git a/src/xrpld/app/consensus/RCLConsensus.h b/src/xrpld/app/consensus/RCLConsensus.h index 38481d23633..e54bae83903 100644 --- a/src/xrpld/app/consensus/RCLConsensus.h +++ b/src/xrpld/app/consensus/RCLConsensus.h @@ -553,12 +553,14 @@ class RclConsensusLogger beast::Journal j_; std::unique_ptr ss_; std::chrono::steady_clock::time_point start_; + std::source_location location_; public: explicit RclConsensusLogger( char const* label, bool validating, - beast::Journal j); + beast::Journal j, + std::source_location location = std::source_location::current()); ~RclConsensusLogger(); std::unique_ptr const& diff --git a/src/xrpld/app/main/Application.cpp b/src/xrpld/app/main/Application.cpp index 05b8f5e5fab..505ded28978 100644 --- a/src/xrpld/app/main/Application.cpp +++ b/src/xrpld/app/main/Application.cpp @@ -69,6 +69,7 @@ #include #include #include +#include #include #include @@ -832,7 +833,10 @@ class ApplicationImp : public Application, public BasicApp serverOkay(std::string& reason) override; beast::Journal - journal(std::string const& name) override; + journal( + std::string const& name, + std::unique_ptr attributes = + {}) override; //-------------------------------------------------------------------------- @@ -1210,8 +1214,15 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline) } JLOG(m_journal.info()) << "Process starting: " - << BuildInfo::getFullVersionString() - << ", Instance Cookie: " << instanceCookie_; + << log::param( + "RippledVersion", + BuildInfo::getFullVersionString()) + << ", Instance Cookie: " + << log::param("InstanceCookie", instanceCookie_); + + Logs::setGlobalAttributes(log::attributes( + {{"RippledVersion", BuildInfo::getFullVersionString()}, + {"InstanceCookie", to_string(instanceCookie_)}})); if (numberOfThreads(*config_) < 2) { @@ -2163,9 +2174,11 @@ ApplicationImp::serverOkay(std::string& reason) } beast::Journal -ApplicationImp::journal(std::string const& name) +ApplicationImp::journal( + std::string const& name, + std::unique_ptr attributes) { - return logs_->journal(name); + return logs_->journal(name, std::move(attributes)); } void diff --git a/src/xrpld/app/main/Application.h b/src/xrpld/app/main/Application.h index b3a433fee8d..f2883a4c394 100644 --- a/src/xrpld/app/main/Application.h +++ b/src/xrpld/app/main/Application.h @@ -258,7 +258,10 @@ class Application : public beast::PropertyStream::Source serverOkay(std::string& reason) = 0; virtual beast::Journal - journal(std::string const& name) = 0; + journal( + std::string const& name, + std::unique_ptr attributes = + {}) = 0; /* Returns the number of file descriptors the application needs */ virtual int diff --git a/src/xrpld/app/main/Main.cpp b/src/xrpld/app/main/Main.cpp index 2353d7acd1d..4e34164eb54 100644 --- a/src/xrpld/app/main/Main.cpp +++ b/src/xrpld/app/main/Main.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -794,6 +795,14 @@ run(int argc, char** argv) else if (vm.count("verbose")) thresh = kTrace; + if (config->LOG_STYLE == LogStyle::Json) + { + static log::JsonStructuredJournal structuredJournal; + beast::Journal::enableStructuredJournal(&structuredJournal); + Logs::setGlobalAttributes(log::attributes( + {{"Application", "rippled"}, {"NetworkID", config->NETWORK_ID}})); + } + auto logs = std::make_unique(thresh); // No arguments. Run server. diff --git a/src/xrpld/app/tx/detail/Transactor.cpp b/src/xrpld/app/tx/detail/Transactor.cpp index fd396e45560..f9fb1b5b60b 100644 --- a/src/xrpld/app/tx/detail/Transactor.cpp +++ b/src/xrpld/app/tx/detail/Transactor.cpp @@ -207,9 +207,12 @@ preflight2(PreflightContext const& ctx) Transactor::Transactor(ApplyContext& ctx) : ctx_(ctx) - , sink_(ctx.journal, to_short_string(ctx.tx.getTransactionID()) + " ") - , j_(sink_) , account_(ctx.tx.getAccountID(sfAccount)) + , j_(ctx.journal, + log::attributes({ + {"TransactionID", to_string(ctx_.tx.getTransactionID())}, + {"AccountID", to_string(account_)}, + })) { } diff --git a/src/xrpld/app/tx/detail/Transactor.h b/src/xrpld/app/tx/detail/Transactor.h index e94b93523da..89feeb8933a 100644 --- a/src/xrpld/app/tx/detail/Transactor.h +++ b/src/xrpld/app/tx/detail/Transactor.h @@ -139,13 +139,13 @@ class Transactor { protected: ApplyContext& ctx_; - beast::WrappedSink sink_; - beast::Journal const j_; AccountID const account_; XRPAmount mPriorBalance; // Balance before fees. XRPAmount mSourceBalance; // Balance after fees. + beast::Journal const j_; + virtual ~Transactor() = default; Transactor(Transactor const&) = delete; Transactor& diff --git a/src/xrpld/core/Config.h b/src/xrpld/core/Config.h index a58867958b1..c4b26694643 100644 --- a/src/xrpld/core/Config.h +++ b/src/xrpld/core/Config.h @@ -25,6 +25,7 @@ #include #include #include // VFALCO Breaks levelization +#include #include // VFALCO FIX: This include should not be here @@ -77,6 +78,16 @@ struct FeeSetup * values.) */ }; +/** + * We support producing plain text logs and structured json logs. + */ +namespace LogStyle { +enum LogStyle { LogFmt, Json }; + +LogStyle +fromString(std::string const&); +}; // namespace LogStyle + // This entire derived class is deprecated. // For new config information use the style implied // in the base class. For existing config information @@ -299,6 +310,9 @@ class Config : public BasicConfig std::optional VALIDATOR_LIST_THRESHOLD; + // Set it to LogStyle::Json to get structured json logs. + LogStyle::LogStyle LOG_STYLE = LogStyle::LogFmt; + public: Config(); diff --git a/src/xrpld/core/ConfigSections.h b/src/xrpld/core/ConfigSections.h index 59af2bcf67b..b896cf8c31e 100644 --- a/src/xrpld/core/ConfigSections.h +++ b/src/xrpld/core/ConfigSections.h @@ -48,6 +48,7 @@ struct ConfigSection #define SECTION_CLUSTER_NODES "cluster_nodes" #define SECTION_COMPRESSION "compression" #define SECTION_DEBUG_LOGFILE "debug_logfile" +#define SECTION_LOG_STYLE "log_style" #define SECTION_ELB_SUPPORT "elb_support" #define SECTION_FEE_DEFAULT "fee_default" #define SECTION_FETCH_DEPTH "fetch_depth" diff --git a/src/xrpld/core/detail/Config.cpp b/src/xrpld/core/detail/Config.cpp index 95147e23d5f..f5c99044106 100644 --- a/src/xrpld/core/detail/Config.cpp +++ b/src/xrpld/core/detail/Config.cpp @@ -690,6 +690,9 @@ Config::loadFromString(std::string const& fileContents) if (getSingleSection(secConfig, SECTION_DEBUG_LOGFILE, strTemp, j_)) DEBUG_LOGFILE = strTemp; + if (getSingleSection(secConfig, SECTION_LOG_STYLE, strTemp, j_)) + LOG_STYLE = LogStyle::fromString(strTemp); + if (getSingleSection(secConfig, SECTION_SWEEP_INTERVAL, strTemp, j_)) { SWEEP_INTERVAL = beast::lexicalCastThrow(strTemp); @@ -1078,6 +1081,16 @@ Config::loadFromString(std::string const& fileContents) } } +LogStyle::LogStyle +LogStyle::fromString(std::string const& str) +{ + if (str == "json") + { + return Json; + } + return LogFmt; +} + boost::filesystem::path Config::getDebugLogFile() const { diff --git a/src/xrpld/overlay/detail/ConnectAttempt.cpp b/src/xrpld/overlay/detail/ConnectAttempt.cpp index c1bc4bb069e..048933c82cd 100644 --- a/src/xrpld/overlay/detail/ConnectAttempt.cpp +++ b/src/xrpld/overlay/detail/ConnectAttempt.cpp @@ -41,8 +41,7 @@ ConnectAttempt::ConnectAttempt( : Child(overlay) , app_(app) , id_(id) - , sink_(journal, OverlayImpl::makePrefix(id)) - , journal_(sink_) + , journal_(journal, log::attributes({{"NodeID", id}})) , remote_endpoint_(remote_endpoint) , usage_(usage) , strand_(boost::asio::make_strand(io_context)) diff --git a/src/xrpld/overlay/detail/ConnectAttempt.h b/src/xrpld/overlay/detail/ConnectAttempt.h index 38b9482d9d9..24fbbb56cf3 100644 --- a/src/xrpld/overlay/detail/ConnectAttempt.h +++ b/src/xrpld/overlay/detail/ConnectAttempt.h @@ -117,7 +117,6 @@ class ConnectAttempt : public OverlayImpl::Child, // Core application and networking components Application& app_; Peer::id_t const id_; - beast::WrappedSink sink_; beast::Journal const journal_; endpoint_type remote_endpoint_; Resource::Consumer usage_; diff --git a/src/xrpld/overlay/detail/OverlayImpl.cpp b/src/xrpld/overlay/detail/OverlayImpl.cpp index 8d295faace7..ee3aa53d7ae 100644 --- a/src/xrpld/overlay/detail/OverlayImpl.cpp +++ b/src/xrpld/overlay/detail/OverlayImpl.cpp @@ -168,8 +168,7 @@ OverlayImpl::onHandoff( endpoint_type remote_endpoint) { auto const id = next_id_++; - beast::WrappedSink sink(app_.logs()["Peer"], makePrefix(id)); - beast::Journal journal(sink); + auto journal = app_.journal("Peer", log::attributes({{"NodeID", id}})); Handoff handoff; if (processRequest(request, handoff)) @@ -337,14 +336,6 @@ OverlayImpl::isPeerUpgrade(http_request_type const& request) return !versions.empty(); } -std::string -OverlayImpl::makePrefix(std::uint32_t id) -{ - std::stringstream ss; - ss << "[" << std::setfill('0') << std::setw(3) << id << "] "; - return ss.str(); -} - std::shared_ptr OverlayImpl::makeRedirectResponse( std::shared_ptr const& slot, diff --git a/src/xrpld/overlay/detail/OverlayImpl.h b/src/xrpld/overlay/detail/OverlayImpl.h index b4ea3307ec7..b8ef7f3fa09 100644 --- a/src/xrpld/overlay/detail/OverlayImpl.h +++ b/src/xrpld/overlay/detail/OverlayImpl.h @@ -344,9 +344,6 @@ class OverlayImpl : public Overlay, public reduce_relay::SquelchHandler return true; } - static std::string - makePrefix(std::uint32_t id); - void reportInboundTraffic(TrafficCount::category cat, int bytes); diff --git a/src/xrpld/overlay/detail/PeerImp.cpp b/src/xrpld/overlay/detail/PeerImp.cpp index 93371f42abb..1b6e611c9e5 100644 --- a/src/xrpld/overlay/detail/PeerImp.cpp +++ b/src/xrpld/overlay/detail/PeerImp.cpp @@ -82,10 +82,18 @@ PeerImp::PeerImp( : Child(overlay) , app_(app) , id_(id) - , sink_(app_.journal("Peer"), makePrefix(id)) - , p_sink_(app_.journal("Protocol"), makePrefix(id)) - , journal_(sink_) - , p_journal_(p_sink_) + , journal_( + app_.journal("Peer"), + log::attributes( + {{"NodeID", id}, + {"RemoteAddress", to_string(slot->remote_endpoint())}, + {"PublicKey", toBase58(TokenType::NodePublic, publicKey)}})) + , p_journal_( + app_.journal("Protocol"), + log::attributes( + {{"NodeID", id}, + {"RemoteAddress", to_string(slot->remote_endpoint())}, + {"PublicKey", toBase58(TokenType::NodePublic, publicKey)}})) , stream_ptr_(std::move(stream_ptr)) , socket_(stream_ptr_->next_layer().socket()) , stream_(*stream_ptr_) diff --git a/src/xrpld/overlay/detail/PeerImp.h b/src/xrpld/overlay/detail/PeerImp.h index c2221c136db..feda39031eb 100644 --- a/src/xrpld/overlay/detail/PeerImp.h +++ b/src/xrpld/overlay/detail/PeerImp.h @@ -134,8 +134,6 @@ class PeerImp : public Peer, Application& app_; id_t const id_; - beast::WrappedSink sink_; - beast::WrappedSink p_sink_; beast::Journal const journal_; beast::Journal const p_journal_; std::unique_ptr stream_ptr_; @@ -634,9 +632,6 @@ class PeerImp : public Peer, void cancelTimer() noexcept; - static std::string - makePrefix(id_t id); - void doAccept(); @@ -832,10 +827,18 @@ PeerImp::PeerImp( : Child(overlay) , app_(app) , id_(id) - , sink_(app_.journal("Peer"), makePrefix(id)) - , p_sink_(app_.journal("Protocol"), makePrefix(id)) - , journal_(sink_) - , p_journal_(p_sink_) + , journal_( + app_.journal("Peer"), + log::attributes( + {{"NodeID", id}, + {"RemoteAddress", to_string(slot->remote_endpoint())}, + {"PublicKey", toBase58(TokenType::NodePublic, publicKey)}})) + , p_journal_( + app_.journal("Protocol"), + log::attributes( + {{"NodeID", id}, + {"RemoteAddress", to_string(slot->remote_endpoint())}, + {"PublicKey", toBase58(TokenType::NodePublic, publicKey)}})) , stream_ptr_(std::move(stream_ptr)) , socket_(stream_ptr_->next_layer().socket()) , stream_(*stream_ptr_) From 1774769226164e864a92548a50cf5ed3e365fc66 Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 21 Aug 2025 14:41:56 +0100 Subject: [PATCH 02/88] Support structured logs Signed-off-by: JCW --- src/libxrpl/logging/JsonLogs.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libxrpl/logging/JsonLogs.cpp b/src/libxrpl/logging/JsonLogs.cpp index 869b6ed09e7..4f7d81f934b 100644 --- a/src/libxrpl/logging/JsonLogs.cpp +++ b/src/libxrpl/logging/JsonLogs.cpp @@ -17,7 +17,9 @@ */ //============================================================================== -#include +#include +#include + namespace ripple { namespace log { @@ -100,14 +102,14 @@ JsonStructuredJournal::Logger::write( threadIdStream << std::this_thread::get_id(); globalContext["ThreadId"] = threadIdStream.str(); globalContext["Params"] = messageParams; - globalContext["Level"] = Logs::toString(Logs::fromSeverity(level)); + globalContext["Level"] = beast::severities::to_string(level); globalContext["Message"] = text; globalContext["Time"] = to_string(std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count()); - sink->write(level, Json::jsonAsString(globalContext)); + sink->write(level, to_string(globalContext)); } JsonStructuredJournal::Logger From f6d7b90b7006b1da014abd359a5534c93c0cd249 Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 21 Aug 2025 14:46:53 +0100 Subject: [PATCH 03/88] Support structured logs Signed-off-by: JCW --- src/libxrpl/telemetry/JsonLogs.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libxrpl/telemetry/JsonLogs.cpp b/src/libxrpl/telemetry/JsonLogs.cpp index 9c9b8762d8e..4ae8028126a 100644 --- a/src/libxrpl/telemetry/JsonLogs.cpp +++ b/src/libxrpl/telemetry/JsonLogs.cpp @@ -20,6 +20,8 @@ #include #include +#include + namespace ripple::log { thread_local JsonStructuredJournal::Logger From af9dde4f75e93fdfea43f06ae0a64866bf328862 Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 21 Aug 2025 14:47:41 +0100 Subject: [PATCH 04/88] Fix formatting Signed-off-by: JCW --- src/libxrpl/logging/JsonLogs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libxrpl/logging/JsonLogs.cpp b/src/libxrpl/logging/JsonLogs.cpp index 4f7d81f934b..b447f1796bf 100644 --- a/src/libxrpl/logging/JsonLogs.cpp +++ b/src/libxrpl/logging/JsonLogs.cpp @@ -17,8 +17,8 @@ */ //============================================================================== -#include #include +#include namespace ripple { namespace log { From e4db80f61d2c120e3b4c2de46d1fe4cd33b1dc0f Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 21 Aug 2025 14:57:39 +0100 Subject: [PATCH 05/88] Fix errors Signed-off-by: JCW --- .../scripts/levelization/results/ordering.txt | 10 +- include/xrpl/logging/JsonLogs.h | 223 ------------------ include/xrpl/telemetry/JsonLogs.h | 7 +- src/libxrpl/logging/JsonLogs.cpp | 138 ----------- 4 files changed, 11 insertions(+), 367 deletions(-) delete mode 100644 include/xrpl/logging/JsonLogs.h delete mode 100644 src/libxrpl/logging/JsonLogs.cpp diff --git a/.github/scripts/levelization/results/ordering.txt b/.github/scripts/levelization/results/ordering.txt index 55df4c26724..7bc289c5c88 100644 --- a/.github/scripts/levelization/results/ordering.txt +++ b/.github/scripts/levelization/results/ordering.txt @@ -62,6 +62,7 @@ test.csf > xrpl.basics test.csf > xrpld.consensus test.csf > xrpl.json test.csf > xrpl.protocol +test.csf > xrpl.telemetry test.json > test.jtx test.json > xrpl.json test.jtx > xrpl.basics @@ -138,7 +139,6 @@ test.toplevel > test.csf test.toplevel > xrpl.json test.unit_test > xrpl.basics tests.libxrpl > xrpl.basics -tests.libxrpl > xrpl.net xrpl.json > xrpl.basics xrpl.ledger > xrpl.basics xrpl.ledger > xrpl.protocol @@ -148,9 +148,12 @@ xrpl.protocol > xrpl.json xrpl.resource > xrpl.basics xrpl.resource > xrpl.json xrpl.resource > xrpl.protocol +xrpl.resource > xrpl.telemetry xrpl.server > xrpl.basics xrpl.server > xrpl.json xrpl.server > xrpl.protocol +xrpl.server > xrpl.telemetry +xrpl.telemetry > xrpl.json xrpld.app > test.unit_test xrpld.app > xrpl.basics xrpld.app > xrpld.conditions @@ -162,6 +165,7 @@ xrpld.app > xrpl.ledger xrpld.app > xrpl.net xrpld.app > xrpl.protocol xrpld.app > xrpl.resource +xrpld.app > xrpl.telemetry xrpld.conditions > xrpl.basics xrpld.conditions > xrpl.protocol xrpld.consensus > xrpl.basics @@ -171,6 +175,10 @@ xrpld.core > xrpl.basics xrpld.core > xrpl.json xrpld.core > xrpl.net xrpld.core > xrpl.protocol +xrpld.core > xrpl.telemetry +xrpld.ledger > xrpl.basics +xrpld.ledger > xrpl.json +xrpld.ledger > xrpl.protocol xrpld.nodestore > xrpl.basics xrpld.nodestore > xrpld.core xrpld.nodestore > xrpld.unity diff --git a/include/xrpl/logging/JsonLogs.h b/include/xrpl/logging/JsonLogs.h deleted file mode 100644 index 1eb2ba8c8a2..00000000000 --- a/include/xrpl/logging/JsonLogs.h +++ /dev/null @@ -1,223 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of Beast: https://github.com/vinniefalco/Beast - Copyright 2013, Vinnie Falco - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_LOGGING_STRUCTUREDJOURNAL_H_INCLUDED -#define RIPPLE_LOGGING_STRUCTUREDJOURNAL_H_INCLUDED - -#include -#include -#include -#include - -#include -#include -#include - -namespace ripple { -namespace log { - -template -class LogParameter -{ -public: - template - LogParameter(char const* name, TArg&& value) - : name_(name), value_(std::forward(value)) - { - } - -private: - char const* name_; - T value_; - - template - friend std::ostream& - operator<<(std::ostream& os, LogParameter const&); -}; - -template -class LogField -{ -public: - template - LogField(char const* name, TArg&& value) - : name_(name), value_(std::forward(value)) - { - } - -private: - char const* name_; - T value_; - - template - friend std::ostream& - operator<<(std::ostream& os, LogField const&); -}; - -class JsonLogAttributes : public beast::Journal::StructuredLogAttributes -{ -public: - using AttributeFields = std::unordered_map; - using Pair = AttributeFields::value_type; - - explicit JsonLogAttributes(AttributeFields contextValues = {}); - - void - setModuleName(std::string const& name) override; - - std::unique_ptr - clone() const override; - - void - combine(std::unique_ptr const& context) override; - - void - combine(std::unique_ptr&& context) override; - - AttributeFields& - contextValues() - { - return contextValues_; - } - -private: - AttributeFields contextValues_; -}; - -class JsonStructuredJournal : public beast::Journal::StructuredJournalImpl -{ -private: - struct Logger - { - std::source_location location = {}; - Json::Value messageParams; - - Logger() = default; - Logger( - JsonStructuredJournal const* journal, - std::source_location location); - - void - write( - beast::Journal::Sink* sink, - beast::severities::Severity level, - std::string const& text, - beast::Journal::StructuredLogAttributes* context) const; - }; - - [[nodiscard]] Logger - logger(std::source_location location) const; - - static thread_local Logger currentLogger_; - - template - friend std::ostream& - operator<<(std::ostream& os, LogParameter const&); - - template - friend std::ostream& - operator<<(std::ostream& os, LogField const&); - -public: - void - initMessageContext(std::source_location location) override; - - void - flush( - beast::Journal::Sink* sink, - beast::severities::Severity level, - std::string const& text, - beast::Journal::StructuredLogAttributes* context) override; -}; - -template -std::ostream& -operator<<(std::ostream& os, LogParameter const& param) -{ - using ValueType = std::decay_t; - // TODO: Update the Json library to support 64-bit integer values. - if constexpr ( - std::constructible_from && - (!std::is_integral_v || - sizeof(ValueType) <= sizeof(Json::Int))) - { - JsonStructuredJournal::currentLogger_.messageParams[param.name_] = - Json::Value{param.value_}; - return os << param.value_; - } - else - { - std::ostringstream oss; - oss << param.value_; - - JsonStructuredJournal::currentLogger_.messageParams[param.name_] = - oss.str(); - return os << oss.str(); - } -} - -template -std::ostream& -operator<<(std::ostream& os, LogField const& param) -{ - using ValueType = std::decay_t; - // TODO: Update the Json library to support 64-bit integer values. - if constexpr ( - std::constructible_from && - (!std::is_integral_v || - sizeof(ValueType) <= sizeof(Json::Int))) - { - JsonStructuredJournal::currentLogger_.messageParams[param.name_] = - Json::Value{param.value_}; - } - else - { - std::ostringstream oss; - oss << param.value_; - - JsonStructuredJournal::currentLogger_.messageParams[param.name_] = - oss.str(); - } - return os; -} - -template -LogParameter -param(char const* name, T&& value) -{ - return LogParameter{name, std::forward(value)}; -} - -template -LogField -field(char const* name, T&& value) -{ - return LogField{name, std::forward(value)}; -} - -[[nodiscard]] inline std::unique_ptr -attributes(std::initializer_list const& fields) -{ - return std::make_unique(fields); -} - -} // namespace log -} // namespace ripple - -#endif diff --git a/include/xrpl/telemetry/JsonLogs.h b/include/xrpl/telemetry/JsonLogs.h index df8fe47625a..8ff996f41f2 100644 --- a/include/xrpl/telemetry/JsonLogs.h +++ b/include/xrpl/telemetry/JsonLogs.h @@ -23,12 +23,10 @@ #include #include -#include #include #include -namespace ripple { -namespace log { +namespace ripple::log { template class LogParameter @@ -215,7 +213,6 @@ attributes(std::initializer_list const& fields) return std::make_unique(fields); } -} // namespace log -} // namespace ripple +} // namespace ripple::log #endif diff --git a/src/libxrpl/logging/JsonLogs.cpp b/src/libxrpl/logging/JsonLogs.cpp deleted file mode 100644 index b447f1796bf..00000000000 --- a/src/libxrpl/logging/JsonLogs.cpp +++ /dev/null @@ -1,138 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of Beast: https://github.com/vinniefalco/Beast - Copyright 2013, Vinnie Falco - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include - -namespace ripple { -namespace log { - -thread_local JsonStructuredJournal::Logger - JsonStructuredJournal::currentLogger_{}; - -JsonLogAttributes::JsonLogAttributes(AttributeFields contextValues) - : contextValues_(std::move(contextValues)) -{ -} - -void -JsonLogAttributes::setModuleName(std::string const& name) -{ - contextValues()["Module"] = name; -} - -std::unique_ptr -JsonLogAttributes::clone() const -{ - return std::make_unique(*this); -} - -void -JsonLogAttributes::combine( - std::unique_ptr const& context) -{ - auto structuredContext = - static_cast(context.get()); - contextValues_.insert( - structuredContext->contextValues_.begin(), - structuredContext->contextValues_.end()); -} - -void -JsonLogAttributes::combine(std::unique_ptr&& context) -{ - auto structuredContext = - static_cast(context.get()); - - if (contextValues_.empty()) - { - contextValues_ = std::move(structuredContext->contextValues_); - } - else - { - contextValues_.insert( - structuredContext->contextValues_.begin(), - structuredContext->contextValues_.end()); - } -} - -JsonStructuredJournal::Logger::Logger( - JsonStructuredJournal const* journal, - std::source_location location) - : location(location) -{ -} - -void -JsonStructuredJournal::Logger::write( - beast::Journal::Sink* sink, - beast::severities::Severity level, - std::string const& text, - beast::Journal::StructuredLogAttributes* context) const -{ - Json::Value globalContext; - if (context) - { - auto jsonContext = static_cast(context); - for (auto const& [key, value] : jsonContext->contextValues()) - { - globalContext[key] = value; - } - } - globalContext["Function"] = location.function_name(); - globalContext["File"] = location.file_name(); - globalContext["Line"] = location.line(); - std::stringstream threadIdStream; - threadIdStream << std::this_thread::get_id(); - globalContext["ThreadId"] = threadIdStream.str(); - globalContext["Params"] = messageParams; - globalContext["Level"] = beast::severities::to_string(level); - globalContext["Message"] = text; - globalContext["Time"] = - to_string(std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count()); - - sink->write(level, to_string(globalContext)); -} - -JsonStructuredJournal::Logger -JsonStructuredJournal::logger(std::source_location location) const -{ - return Logger{this, location}; -} - -void -JsonStructuredJournal::initMessageContext(std::source_location location) -{ - currentLogger_ = logger(location); -} - -void -JsonStructuredJournal::flush( - beast::Journal::Sink* sink, - beast::severities::Severity level, - std::string const& text, - beast::Journal::StructuredLogAttributes* context) -{ - currentLogger_.write(sink, level, text, context); -} - -} // namespace log -} // namespace ripple From 43c6e202af00a30b755732e2262451a5a2df9d7f Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 21 Aug 2025 15:08:10 +0100 Subject: [PATCH 06/88] Fix to_string error Signed-off-by: JCW --- src/libxrpl/telemetry/JsonLogs.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libxrpl/telemetry/JsonLogs.cpp b/src/libxrpl/telemetry/JsonLogs.cpp index 4ae8028126a..1e159713f8e 100644 --- a/src/libxrpl/telemetry/JsonLogs.cpp +++ b/src/libxrpl/telemetry/JsonLogs.cpp @@ -19,6 +19,7 @@ #include #include +#include #include From eda9bf1f1ad478c3cb4dafa93f5b2ac7b807baa0 Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 21 Aug 2025 15:09:47 +0100 Subject: [PATCH 07/88] Fix formatting Signed-off-by: JCW --- src/libxrpl/telemetry/JsonLogs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libxrpl/telemetry/JsonLogs.cpp b/src/libxrpl/telemetry/JsonLogs.cpp index 1e159713f8e..a7ae2175650 100644 --- a/src/libxrpl/telemetry/JsonLogs.cpp +++ b/src/libxrpl/telemetry/JsonLogs.cpp @@ -17,9 +17,9 @@ */ //============================================================================== +#include #include #include -#include #include From 0e4f9a7ccfc4ae9e996c3f5d44227958af059730 Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 21 Aug 2025 15:16:08 +0100 Subject: [PATCH 08/88] Fix errors Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 1 - include/xrpl/telemetry/JsonLogs.h | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 5942b8de095..1cb5bd1a5c2 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -365,7 +365,6 @@ class Journal /** Journal has no default constructor. */ Journal() = delete; - [[deprecated]] Journal(Journal const& other) : Journal(other, nullptr) { diff --git a/include/xrpl/telemetry/JsonLogs.h b/include/xrpl/telemetry/JsonLogs.h index 8ff996f41f2..c6f3b84ff61 100644 --- a/include/xrpl/telemetry/JsonLogs.h +++ b/include/xrpl/telemetry/JsonLogs.h @@ -25,6 +25,7 @@ #include #include +#include namespace ripple::log { @@ -77,7 +78,7 @@ class JsonLogAttributes : public beast::Journal::StructuredLogAttributes void setModuleName(std::string const& name) override; - std::unique_ptr + [[nodiscard]] std::unique_ptr clone() const override; void From 9edba67e647ae2ffc4e6aa9d4d68d1e834593047 Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 21 Aug 2025 15:16:34 +0100 Subject: [PATCH 09/88] Fix formatting Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 3 +-- include/xrpl/telemetry/JsonLogs.h | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 1cb5bd1a5c2..15a106bd32a 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -365,8 +365,7 @@ class Journal /** Journal has no default constructor. */ Journal() = delete; - Journal(Journal const& other) - : Journal(other, nullptr) + Journal(Journal const& other) : Journal(other, nullptr) { } diff --git a/include/xrpl/telemetry/JsonLogs.h b/include/xrpl/telemetry/JsonLogs.h index c6f3b84ff61..a880184bf7a 100644 --- a/include/xrpl/telemetry/JsonLogs.h +++ b/include/xrpl/telemetry/JsonLogs.h @@ -23,9 +23,9 @@ #include #include +#include #include #include -#include namespace ripple::log { From 856b36d0a564c4e7ed39275005aea8815fdbf0aa Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 21 Aug 2025 15:27:06 +0100 Subject: [PATCH 10/88] Fix errors Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 1 + include/xrpl/telemetry/JsonLogs.h | 1 + 2 files changed, 2 insertions(+) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 15a106bd32a..c02bdb88f91 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -22,6 +22,7 @@ #include +#include #include #include diff --git a/include/xrpl/telemetry/JsonLogs.h b/include/xrpl/telemetry/JsonLogs.h index a880184bf7a..1ed3e16a8dc 100644 --- a/include/xrpl/telemetry/JsonLogs.h +++ b/include/xrpl/telemetry/JsonLogs.h @@ -25,6 +25,7 @@ #include #include +#include #include namespace ripple::log { From 9e09595db02750a9876024fce360ee000adf767e Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 21 Aug 2025 15:34:21 +0100 Subject: [PATCH 11/88] Fix errors Signed-off-by: JCW --- src/libxrpl/beast/utility/beast_Journal.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index de9a5f196fc..1b9cafc5647 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -109,8 +109,10 @@ severities::to_string(Severity severity) case kFatal: return "fatal"; default: - assert(false); + UNREACHABLE("Unexpected severity value!"); } + return ""; + } Journal::Sink::Sink(Severity thresh, bool console) : thresh_(thresh), m_console(console) From 34127593e67e8dfe4979053d0b7c0106bad9872d Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 21 Aug 2025 15:51:50 +0100 Subject: [PATCH 12/88] Fix errors Signed-off-by: JCW --- src/libxrpl/beast/utility/beast_Journal.cpp | 1 - src/libxrpl/telemetry/JsonLogs.cpp | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index 1b9cafc5647..ad31a1a179f 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -112,7 +112,6 @@ severities::to_string(Severity severity) UNREACHABLE("Unexpected severity value!"); } return ""; - } Journal::Sink::Sink(Severity thresh, bool console) : thresh_(thresh), m_console(console) diff --git a/src/libxrpl/telemetry/JsonLogs.cpp b/src/libxrpl/telemetry/JsonLogs.cpp index a7ae2175650..21b1b51adbf 100644 --- a/src/libxrpl/telemetry/JsonLogs.cpp +++ b/src/libxrpl/telemetry/JsonLogs.cpp @@ -21,6 +21,7 @@ #include #include +#include #include namespace ripple::log { From d68f87f968df2c8894132aebdd84d8d6b52fe1f5 Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 21 Aug 2025 16:15:17 +0100 Subject: [PATCH 13/88] Fix errors Signed-off-by: JCW --- src/tests/libxrpl/basics/log.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index 8ce69af68d9..7c3710bfe6d 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -72,10 +72,10 @@ class MockLogs : public Logs { } - virtual std::unique_ptr + std::unique_ptr makeSink( std::string const& partition, - beast::severities::Severity startingLevel) + beast::severities::Severity startingLevel) override { return std::make_unique(partition, startingLevel, *this); } From b3f389d91828f5969b8a3ec9e60b53e7d467d356 Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 21 Aug 2025 16:26:19 +0100 Subject: [PATCH 14/88] Remove unneeded file Signed-off-by: JCW --- src/tests/libxrpl/telemetry/base64.cpp | 67 -------------------------- 1 file changed, 67 deletions(-) delete mode 100644 src/tests/libxrpl/telemetry/base64.cpp diff --git a/src/tests/libxrpl/telemetry/base64.cpp b/src/tests/libxrpl/telemetry/base64.cpp deleted file mode 100644 index fe9b86abb10..00000000000 --- a/src/tests/libxrpl/telemetry/base64.cpp +++ /dev/null @@ -1,67 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include - -#include - -#include - -using namespace ripple; - -static void -check(std::string const& in, std::string const& out) -{ - auto const encoded = base64_encode(in); - CHECK(encoded == out); - CHECK(base64_decode(encoded) == in); -} - -TEST_CASE("base64") -{ - check("", ""); - check("f", "Zg=="); - check("fo", "Zm8="); - check("foo", "Zm9v"); - check("foob", "Zm9vYg=="); - check("fooba", "Zm9vYmE="); - check("foobar", "Zm9vYmFy"); - - check( - "Man is distinguished, not only by his reason, but by this " - "singular passion from " - "other animals, which is a lust of the mind, that by a " - "perseverance of delight " - "in the continued and indefatigable generation of knowledge, " - "exceeds the short " - "vehemence of any carnal pleasure.", - "TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dC" - "BieSB0aGlz" - "IHNpbmd1bGFyIHBhc3Npb24gZnJvbSBvdGhlciBhbmltYWxzLCB3aGljaCBpcyBhIG" - "x1c3Qgb2Yg" - "dGhlIG1pbmQsIHRoYXQgYnkgYSBwZXJzZXZlcmFuY2Ugb2YgZGVsaWdodCBpbiB0aG" - "UgY29udGlu" - "dWVkIGFuZCBpbmRlZmF0aWdhYmxlIGdlbmVyYXRpb24gb2Yga25vd2xlZGdlLCBleG" - "NlZWRzIHRo" - "ZSBzaG9ydCB2ZWhlbWVuY2Ugb2YgYW55IGNhcm5hbCBwbGVhc3VyZS4="); - - std::string const notBase64 = "not_base64!!"; - std::string const truncated = "not"; - CHECK(base64_decode(notBase64) == base64_decode(truncated)); -} From a90bf169bf8f92ab35a01e3526c45d2e0b38ae9f Mon Sep 17 00:00:00 2001 From: JCW Date: Fri, 22 Aug 2025 13:33:09 +0100 Subject: [PATCH 15/88] Improve coverage Signed-off-by: JCW --- include/xrpl/basics/Log.h | 14 +++---- include/xrpl/beast/utility/Journal.h | 6 +++ src/tests/libxrpl/basics/log.cpp | 48 +++++++++++++++++++++-- src/tests/libxrpl/telemetry/json_logs.cpp | 14 ++++++- 4 files changed, 70 insertions(+), 12 deletions(-) diff --git a/include/xrpl/basics/Log.h b/include/xrpl/basics/Log.h index 2c407dda171..8df19045a14 100644 --- a/include/xrpl/basics/Log.h +++ b/include/xrpl/basics/Log.h @@ -256,19 +256,19 @@ class Logs static LogSeverity fromString(std::string const& s); -private: - enum { - // Maximum line length for log messages. - // If the message exceeds this length it will be truncated with elipses. - maximumMessageCharacters = 12 * 1024 - }; - static void format( std::string& output, std::string const& message, beast::severities::Severity severity, std::string const& partition); + +private: + enum { + // Maximum line length for log messages. + // If the message exceeds this length it will be truncated with elipses. + maximumMessageCharacters = 12 * 1024 + }; }; // Wraps a Journal::Stream to skip evaluation of diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index c02bdb88f91..414618016c7 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -90,6 +90,12 @@ class Journal m_structuredJournalImpl = impl; } + static void + disableStructuredJournal() + { + m_structuredJournalImpl = nullptr; + } + static bool isStructuredJournalEnabled() { diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index 7c3710bfe6d..108eb146cd6 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -81,7 +81,46 @@ class MockLogs : public Logs } }; -TEST_CASE("Enable Json Logs") +TEST_CASE("Text logs") +{ + std::stringstream logStream; + + MockLogs logs{logStream, beast::severities::kAll}; + + logs.journal("Test").debug() << "Test"; + + CHECK(logStream.str().find("Test") != std::string::npos); + + logStream.str(""); + + logs.journal("Test").debug() << "\n"; + + CHECK(logStream.str().find("\n") == std::string::npos); +} + + +TEST_CASE("Test format output") +{ + std::string output; + Logs::format(output, "Message", beast::severities::kDebug, "Test"); + CHECK(output.find("Message") != std::string::npos); + CHECK(output != "Message"); +} + +TEST_CASE("Test format output when structured logs are enabled") +{ + static log::JsonStructuredJournal structuredJournal; + beast::Journal::enableStructuredJournal(&structuredJournal); + + std::string output; + Logs::format(output, "Message", beast::severities::kDebug, "Test"); + + CHECK(output == "Message"); + + beast::Journal::disableStructuredJournal(); +} + +TEST_CASE("Enable json logs") { static log::JsonStructuredJournal structuredJournal; @@ -97,7 +136,7 @@ TEST_CASE("Enable Json Logs") beast::Journal::enableStructuredJournal(&structuredJournal); - logs.journal("Test").debug() << "Test"; + logs.journal("Test").debug() << "\n"; Json::Reader reader; Json::Value jsonLog; @@ -108,7 +147,8 @@ TEST_CASE("Enable Json Logs") CHECK(jsonLog.isObject()); CHECK(jsonLog.isMember("Message")); CHECK(jsonLog["Message"].isString()); - CHECK(jsonLog["Message"].asString() == "Test"); + CHECK(jsonLog["Message"].asString() == ""); + beast::Journal::disableStructuredJournal(); } TEST_CASE("Global attributes") @@ -134,6 +174,7 @@ TEST_CASE("Global attributes") CHECK(jsonLog.isMember("Field1")); CHECK(jsonLog["Field1"].isString()); CHECK(jsonLog["Field1"].asString() == "Value1"); + beast::Journal::disableStructuredJournal(); } TEST_CASE("Global attributes inheritable") @@ -166,4 +207,5 @@ TEST_CASE("Global attributes inheritable") CHECK(jsonLog["Field1"].asString() == "Value3"); CHECK(jsonLog["Field2"].isString()); CHECK(jsonLog["Field2"].asString() == "Value2"); + beast::Journal::disableStructuredJournal(); } \ No newline at end of file diff --git a/src/tests/libxrpl/telemetry/json_logs.cpp b/src/tests/libxrpl/telemetry/json_logs.cpp index 209606dc8ca..ac8a80af6dc 100644 --- a/src/tests/libxrpl/telemetry/json_logs.cpp +++ b/src/tests/libxrpl/telemetry/json_logs.cpp @@ -62,6 +62,11 @@ class JsonLogStreamFixture beast::Journal::enableStructuredJournal(&structuredJournal); } + ~JsonLogStreamFixture() + { + beast::Journal::disableStructuredJournal(); + } + std::stringstream& stream() { @@ -82,7 +87,12 @@ class JsonLogStreamFixture TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") { - journal().debug() << "Test"; + journal().debug() << std::boolalpha + << true + << std::noboolalpha + << " Test " + << std::boolalpha + << false; Json::Value logValue; Json::Reader reader; @@ -103,7 +113,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") CHECK(logValue["Line"].isNumeric()); CHECK(logValue["Params"].isNull()); CHECK(logValue["Message"].isString()); - CHECK(logValue["Message"].asString() == "Test"); + CHECK(logValue["Message"].asString() == "true Test false"); } TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") From d36ef0cd181feb57adbbc62588377a69237aba32 Mon Sep 17 00:00:00 2001 From: JCW Date: Fri, 22 Aug 2025 14:26:26 +0100 Subject: [PATCH 16/88] Fix formatting Signed-off-by: JCW --- src/tests/libxrpl/basics/log.cpp | 1 - src/tests/libxrpl/telemetry/json_logs.cpp | 8 ++------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index 108eb146cd6..c4a3bab56e5 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -98,7 +98,6 @@ TEST_CASE("Text logs") CHECK(logStream.str().find("\n") == std::string::npos); } - TEST_CASE("Test format output") { std::string output; diff --git a/src/tests/libxrpl/telemetry/json_logs.cpp b/src/tests/libxrpl/telemetry/json_logs.cpp index ac8a80af6dc..9089fa47ccd 100644 --- a/src/tests/libxrpl/telemetry/json_logs.cpp +++ b/src/tests/libxrpl/telemetry/json_logs.cpp @@ -87,12 +87,8 @@ class JsonLogStreamFixture TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") { - journal().debug() << std::boolalpha - << true - << std::noboolalpha - << " Test " - << std::boolalpha - << false; + journal().debug() << std::boolalpha << true << std::noboolalpha << " Test " + << std::boolalpha << false; Json::Value logValue; Json::Reader reader; From d0f078949091aa1ce76b05928b707f24ccca2631 Mon Sep 17 00:00:00 2001 From: Jingchen Date: Wed, 27 Aug 2025 11:10:30 +0100 Subject: [PATCH 17/88] Update include/xrpl/basics/Log.h Co-authored-by: Vito Tumas <5780819+Tapanito@users.noreply.github.com> --- include/xrpl/basics/Log.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/xrpl/basics/Log.h b/include/xrpl/basics/Log.h index 8df19045a14..b9eaa6b90ce 100644 --- a/include/xrpl/basics/Log.h +++ b/include/xrpl/basics/Log.h @@ -239,7 +239,7 @@ class Logs } else { - globalLogAttributes_->combine(std::move(globalLogAttributes_)); + globalLogAttributes_->combine(std::move(globalLogAttributes)); } } From ab9e6563e4dae7bea66c8065826c6203504ae4f4 Mon Sep 17 00:00:00 2001 From: JCW Date: Wed, 27 Aug 2025 11:58:37 +0100 Subject: [PATCH 18/88] Fix PR comments Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 36 +++------------------ src/libxrpl/basics/Log.cpp | 4 --- src/libxrpl/beast/utility/beast_Journal.cpp | 10 +----- src/libxrpl/telemetry/JsonLogs.cpp | 6 ---- src/tests/libxrpl/basics/log.cpp | 16 ++++----- src/tests/libxrpl/telemetry/json_logs.cpp | 4 +-- src/xrpld/app/main/Main.cpp | 4 +-- src/xrpld/core/detail/Config.cpp | 2 -- 8 files changed, 17 insertions(+), 65 deletions(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 414618016c7..ea4f2771931 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -76,7 +76,7 @@ class Journal std::unique_ptr m_attributes; - static StructuredJournalImpl* m_structuredJournalImpl; + static std::unique_ptr m_structuredJournalImpl; // Invariant: m_sink always points to a valid Sink Sink* m_sink = nullptr; @@ -85,9 +85,9 @@ class Journal //-------------------------------------------------------------------------- static void - enableStructuredJournal(StructuredJournalImpl* impl) + enableStructuredJournal(std::unique_ptr impl) { - m_structuredJournalImpl = impl; + m_structuredJournalImpl = std::move(impl); } static void @@ -99,7 +99,7 @@ class Journal static bool isStructuredJournalEnabled() { - return m_structuredJournalImpl; + return m_structuredJournalImpl != nullptr; } class StructuredJournalImpl @@ -382,19 +382,13 @@ class Journal : m_sink(other.m_sink) { if (attributes) - { m_attributes = std::move(attributes); - } if (other.m_attributes) { if (m_attributes) - { m_attributes->combine(other.m_attributes); - } else - { m_attributes = other.m_attributes->clone(); - } } } @@ -404,19 +398,13 @@ class Journal : m_sink(other.m_sink) { if (attributes) - { m_attributes = std::move(attributes); - } if (other.m_attributes) { if (m_attributes) - { m_attributes->combine(std::move(other.m_attributes)); - } else - { m_attributes = std::move(other.m_attributes); - } } } @@ -439,9 +427,7 @@ class Journal { m_sink = other.m_sink; if (other.m_attributes) - { m_attributes = other.m_attributes->clone(); - } return *this; } @@ -450,9 +436,7 @@ class Journal { m_sink = other.m_sink; if (other.m_attributes) - { m_attributes = std::move(other.m_attributes); - } return *this; } @@ -487,9 +471,7 @@ class Journal trace(std::source_location location = std::source_location::current()) const { if (m_structuredJournalImpl) - { m_structuredJournalImpl->initMessageContext(location); - } return { m_attributes ? m_attributes->clone() : nullptr, *m_sink, @@ -500,9 +482,7 @@ class Journal debug(std::source_location location = std::source_location::current()) const { if (m_structuredJournalImpl) - { m_structuredJournalImpl->initMessageContext(location); - } return { m_attributes ? m_attributes->clone() : nullptr, *m_sink, @@ -513,9 +493,7 @@ class Journal info(std::source_location location = std::source_location::current()) const { if (m_structuredJournalImpl) - { m_structuredJournalImpl->initMessageContext(location); - } return { m_attributes ? m_attributes->clone() : nullptr, *m_sink, @@ -526,9 +504,7 @@ class Journal warn(std::source_location location = std::source_location::current()) const { if (m_structuredJournalImpl) - { m_structuredJournalImpl->initMessageContext(location); - } return { m_attributes ? m_attributes->clone() : nullptr, *m_sink, @@ -539,9 +515,7 @@ class Journal error(std::source_location location = std::source_location::current()) const { if (m_structuredJournalImpl) - { m_structuredJournalImpl->initMessageContext(location); - } return { m_attributes ? m_attributes->clone() : nullptr, *m_sink, @@ -552,9 +526,7 @@ class Journal fatal(std::source_location location = std::source_location::current()) const { if (m_structuredJournalImpl) - { m_structuredJournalImpl->initMessageContext(location); - } return { m_attributes ? m_attributes->clone() : nullptr, *m_sink, diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index eba549d078a..e7a929af89f 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -167,13 +167,9 @@ Logs::journal( if (globalLogAttributes_) { if (attributes) - { attributes->combine(globalLogAttributes_); - } else - { attributes = globalLogAttributes_->clone(); - } } return beast::Journal(get(name), name, std::move(attributes)); } diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index ad31a1a179f..d43c05f0214 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -25,7 +25,7 @@ namespace beast { -Journal::StructuredJournalImpl* Journal::m_structuredJournalImpl = nullptr; +std::unique_ptr Journal::m_structuredJournalImpl; //------------------------------------------------------------------------------ @@ -179,26 +179,18 @@ Journal::ScopedStream::~ScopedStream() if (s == "\n") { if (m_structuredJournalImpl) - { m_structuredJournalImpl->flush( &m_sink, m_level, "", m_attributes.get()); - } else - { m_sink.write(m_level, ""); - } } else { if (m_structuredJournalImpl) - { m_structuredJournalImpl->flush( &m_sink, m_level, s, m_attributes.get()); - } else - { m_sink.write(m_level, s); - } } } } diff --git a/src/libxrpl/telemetry/JsonLogs.cpp b/src/libxrpl/telemetry/JsonLogs.cpp index 21b1b51adbf..1006c8becc6 100644 --- a/src/libxrpl/telemetry/JsonLogs.cpp +++ b/src/libxrpl/telemetry/JsonLogs.cpp @@ -61,13 +61,9 @@ JsonLogAttributes::combine(std::unique_ptr&& context) auto structuredContext = static_cast(context.get()); if (contextValues_.empty()) - { contextValues_ = std::move(structuredContext->contextValues_); - } else - { contextValues_.merge(structuredContext->contextValues_); - } } JsonStructuredJournal::Logger::Logger( @@ -89,9 +85,7 @@ JsonStructuredJournal::Logger::write( { auto jsonContext = static_cast(context); for (auto const& [key, value] : jsonContext->contextValues()) - { globalContext[key] = value; - } } globalContext["Function"] = location.function_name(); globalContext["File"] = location.file_name(); diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index c4a3bab56e5..047bf337322 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -108,8 +108,8 @@ TEST_CASE("Test format output") TEST_CASE("Test format output when structured logs are enabled") { - static log::JsonStructuredJournal structuredJournal; - beast::Journal::enableStructuredJournal(&structuredJournal); + auto structuredJournal = std::make_unique(); + beast::Journal::enableStructuredJournal(std::move(structuredJournal)); std::string output; Logs::format(output, "Message", beast::severities::kDebug, "Test"); @@ -121,7 +121,7 @@ TEST_CASE("Test format output when structured logs are enabled") TEST_CASE("Enable json logs") { - static log::JsonStructuredJournal structuredJournal; + auto structuredJournal = std::make_unique(); std::stringstream logStream; @@ -133,7 +133,7 @@ TEST_CASE("Enable json logs") logStream.str(""); - beast::Journal::enableStructuredJournal(&structuredJournal); + beast::Journal::enableStructuredJournal(std::move(structuredJournal)); logs.journal("Test").debug() << "\n"; @@ -152,13 +152,13 @@ TEST_CASE("Enable json logs") TEST_CASE("Global attributes") { - static log::JsonStructuredJournal structuredJournal; + auto structuredJournal = std::make_unique(); std::stringstream logStream; MockLogs logs{logStream, beast::severities::kAll}; - beast::Journal::enableStructuredJournal(&structuredJournal); + beast::Journal::enableStructuredJournal(std::move(structuredJournal)); MockLogs::setGlobalAttributes(log::attributes({{"Field1", "Value1"}})); logs.journal("Test").debug() << "Test"; @@ -178,13 +178,13 @@ TEST_CASE("Global attributes") TEST_CASE("Global attributes inheritable") { - static log::JsonStructuredJournal structuredJournal; + auto structuredJournal = std::make_unique(); std::stringstream logStream; MockLogs logs{logStream, beast::severities::kAll}; - beast::Journal::enableStructuredJournal(&structuredJournal); + beast::Journal::enableStructuredJournal(std::move(structuredJournal)); MockLogs::setGlobalAttributes(log::attributes({{"Field1", "Value1"}})); logs.journal( diff --git a/src/tests/libxrpl/telemetry/json_logs.cpp b/src/tests/libxrpl/telemetry/json_logs.cpp index 9089fa47ccd..9d95959efe7 100644 --- a/src/tests/libxrpl/telemetry/json_logs.cpp +++ b/src/tests/libxrpl/telemetry/json_logs.cpp @@ -58,8 +58,8 @@ class JsonLogStreamFixture JsonLogStreamFixture() : sink_(beast::severities::kAll, logStream_), j_(sink_) { - static log::JsonStructuredJournal structuredJournal; - beast::Journal::enableStructuredJournal(&structuredJournal); + auto structuredJournal = std::make_unique(); + beast::Journal::enableStructuredJournal(std::move(structuredJournal)); } ~JsonLogStreamFixture() diff --git a/src/xrpld/app/main/Main.cpp b/src/xrpld/app/main/Main.cpp index 4e34164eb54..1de6d0258e8 100644 --- a/src/xrpld/app/main/Main.cpp +++ b/src/xrpld/app/main/Main.cpp @@ -797,8 +797,8 @@ run(int argc, char** argv) if (config->LOG_STYLE == LogStyle::Json) { - static log::JsonStructuredJournal structuredJournal; - beast::Journal::enableStructuredJournal(&structuredJournal); + auto structuredJournal = std::make_unique(); + beast::Journal::enableStructuredJournal(std::move(structuredJournal)); Logs::setGlobalAttributes(log::attributes( {{"Application", "rippled"}, {"NetworkID", config->NETWORK_ID}})); } diff --git a/src/xrpld/core/detail/Config.cpp b/src/xrpld/core/detail/Config.cpp index f5c99044106..7eb3a68a46d 100644 --- a/src/xrpld/core/detail/Config.cpp +++ b/src/xrpld/core/detail/Config.cpp @@ -1085,9 +1085,7 @@ LogStyle::LogStyle LogStyle::fromString(std::string const& str) { if (str == "json") - { return Json; - } return LogFmt; } From 0fe8f3f62d0c123ee8da6aaef797e09ac9481fbd Mon Sep 17 00:00:00 2001 From: JCW Date: Wed, 27 Aug 2025 12:06:29 +0100 Subject: [PATCH 19/88] Hardcode the log style as json Signed-off-by: JCW --- src/xrpld/core/detail/Config.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/xrpld/core/detail/Config.cpp b/src/xrpld/core/detail/Config.cpp index 7eb3a68a46d..726721abd46 100644 --- a/src/xrpld/core/detail/Config.cpp +++ b/src/xrpld/core/detail/Config.cpp @@ -690,8 +690,7 @@ Config::loadFromString(std::string const& fileContents) if (getSingleSection(secConfig, SECTION_DEBUG_LOGFILE, strTemp, j_)) DEBUG_LOGFILE = strTemp; - if (getSingleSection(secConfig, SECTION_LOG_STYLE, strTemp, j_)) - LOG_STYLE = LogStyle::fromString(strTemp); + LOG_STYLE = LogStyle::Json; if (getSingleSection(secConfig, SECTION_SWEEP_INTERVAL, strTemp, j_)) { From cdf11095587e6bfa49c317102db04afb239e422d Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 28 Aug 2025 20:36:46 +0100 Subject: [PATCH 20/88] Performance improvement --- cmake/RippledCore.cmake | 11 +- cmake/RippledInstall.cmake | 1 - conanfile.py | 1 + include/xrpl/basics/Log.h | 20 +- include/xrpl/beast/utility/Journal.h | 430 ++++++++++++++----- include/xrpl/resource/detail/Logic.h | 1 - include/xrpl/server/detail/BasePeer.h | 5 +- include/xrpl/telemetry/JsonLogs.h | 220 ---------- src/libxrpl/basics/Log.cpp | 14 +- src/libxrpl/beast/utility/beast_Journal.cpp | 211 +++++++++- src/libxrpl/telemetry/JsonLogs.cpp | 129 ------ src/test/core/Coroutine_test.cpp | 66 ++- src/test/csf/Peer.h | 3 +- src/tests/libxrpl/CMakeLists.txt | 3 +- src/tests/libxrpl/basics/log.cpp | 431 ++++++++++++++++++-- src/tests/libxrpl/telemetry/json_logs.cpp | 359 ---------------- src/tests/libxrpl/telemetry/main.cpp | 2 - src/xrpld/app/consensus/RCLConsensus.cpp | 2 +- src/xrpld/app/main/Application.cpp | 13 +- src/xrpld/app/main/Application.h | 4 +- src/xrpld/app/main/Main.cpp | 9 +- src/xrpld/app/misc/NetworkOPs.cpp | 7 + src/xrpld/app/tx/detail/Transactor.cpp | 8 +- src/xrpld/app/tx/detail/Transactor.h | 12 +- src/xrpld/core/ClosureCounter.h | 7 + src/xrpld/core/Config.h | 1 - src/xrpld/core/Coro.ipp | 8 + src/xrpld/core/detail/JobQueue.cpp | 4 +- src/xrpld/overlay/detail/ConnectAttempt.cpp | 2 +- src/xrpld/overlay/detail/OverlayImpl.cpp | 2 +- src/xrpld/overlay/detail/PeerImp.cpp | 18 +- src/xrpld/overlay/detail/PeerImp.h | 12 +- 32 files changed, 1048 insertions(+), 968 deletions(-) delete mode 100644 include/xrpl/telemetry/JsonLogs.h delete mode 100644 src/libxrpl/telemetry/JsonLogs.cpp delete mode 100644 src/tests/libxrpl/telemetry/json_logs.cpp delete mode 100644 src/tests/libxrpl/telemetry/main.cpp diff --git a/cmake/RippledCore.cmake b/cmake/RippledCore.cmake index 0e7ab8be211..b9d5cb2cc60 100644 --- a/cmake/RippledCore.cmake +++ b/cmake/RippledCore.cmake @@ -51,6 +51,8 @@ target_link_libraries(xrpl.libpb # TODO: Clean up the number of library targets later. add_library(xrpl.imports.main INTERFACE) +find_package(RapidJSON) + target_link_libraries(xrpl.imports.main INTERFACE LibArchive::LibArchive @@ -75,6 +77,7 @@ add_module(xrpl beast) target_link_libraries(xrpl.libxrpl.beast PUBLIC xrpl.imports.main xrpl.libpb + rapidjson ) # Level 02 @@ -85,14 +88,9 @@ target_link_libraries(xrpl.libxrpl.basics PUBLIC xrpl.libxrpl.beast) add_module(xrpl json) target_link_libraries(xrpl.libxrpl.json PUBLIC xrpl.libxrpl.basics) -add_module(xrpl telemetry) -target_link_libraries(xrpl.libxrpl.telemetry PUBLIC xrpl.libxrpl.json) add_module(xrpl crypto) -target_link_libraries(xrpl.libxrpl.crypto PUBLIC - xrpl.libxrpl.basics - xrpl.libxrpl.telemetry -) +target_link_libraries(xrpl.libxrpl.crypto PUBLIC xrpl.libxrpl.basics) # Level 04 add_module(xrpl protocol) @@ -139,7 +137,6 @@ target_link_modules(xrpl PUBLIC beast crypto json - telemetry protocol resource server diff --git a/cmake/RippledInstall.cmake b/cmake/RippledInstall.cmake index 48fb0fcf954..95c25a212f9 100644 --- a/cmake/RippledInstall.cmake +++ b/cmake/RippledInstall.cmake @@ -16,7 +16,6 @@ install ( xrpl.libxrpl.beast xrpl.libxrpl.crypto xrpl.libxrpl.json - xrpl.libxrpl.telemetry xrpl.libxrpl.protocol xrpl.libxrpl.resource xrpl.libxrpl.ledger diff --git a/conanfile.py b/conanfile.py index 3146b887e0d..024ef85c0dc 100644 --- a/conanfile.py +++ b/conanfile.py @@ -30,6 +30,7 @@ class Xrpl(ConanFile): 'openssl/1.1.1w', 'soci/4.0.3', 'zlib/1.3.1', + "rapidjson/1.1.0" ] test_requires = [ diff --git a/include/xrpl/basics/Log.h b/include/xrpl/basics/Log.h index b9eaa6b90ce..6638da7bee1 100644 --- a/include/xrpl/basics/Log.h +++ b/include/xrpl/basics/Log.h @@ -167,8 +167,6 @@ class Logs beast::severities::Severity thresh_; File file_; bool silent_ = false; - static std::unique_ptr - globalLogAttributes_; public: Logs(beast::severities::Severity level); @@ -191,8 +189,8 @@ class Logs beast::Journal journal( std::string const& name, - std::unique_ptr attributes = - {}); + std::optional attributes = + std::nullopt); beast::severities::Severity threshold() const; @@ -229,20 +227,6 @@ class Logs std::string const& partition, beast::severities::Severity startingLevel); - static void - setGlobalAttributes(std::unique_ptr - globalLogAttributes) - { - if (!globalLogAttributes_) - { - globalLogAttributes_ = std::move(globalLogAttributes); - } - else - { - globalLogAttributes_->combine(std::move(globalLogAttributes)); - } - } - public: static LogSeverity fromSeverity(beast::severities::Severity level); diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index ea4f2771931..6a232cf57c1 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -21,11 +21,62 @@ #define BEAST_UTILITY_JOURNAL_H_INCLUDED #include +#include -#include +#include +#include #include #include + +namespace ripple::log { +template +class LogParameter +{ +public: + template + LogParameter(char const* name, TArg&& value) + : name_(name), value_(std::forward(value)) + { + } + +private: + char const* name_; + T value_; + + template + friend std::ostream& + operator<<(std::ostream& os, LogParameter const&); +}; + +template +class LogField +{ +public: + template + LogField(char const* name, TArg&& value) + : name_(name), value_(std::forward(value)) + { + } + +private: + char const* name_; + T value_; + + template + friend std::ostream& + operator<<(std::ostream& os, LogField const&); +}; + +template +std::ostream& +operator<<(std::ostream& os, LogField const& param); + +template +std::ostream& +operator<<(std::ostream& os, LogParameter const& param); +} + namespace beast { /** A namespace for easy access to logging severity values. */ @@ -64,75 +115,111 @@ to_string(Severity severity); class Journal { public: + template + friend std::ostream& + ripple::log::operator<<(std::ostream& os, ripple::log::LogField const& param); + + template + friend std::ostream& + ripple::log::operator<<( + std::ostream& os, + ripple::log::LogParameter const& param); + class Sink; - class StructuredJournalImpl; + class JsonLogAttributes + { + public: + using AttributeFields = rapidjson::Value; + + JsonLogAttributes(); + JsonLogAttributes(JsonLogAttributes const& other); + + JsonLogAttributes& + operator=(JsonLogAttributes const& other); + + void + setModuleName(std::string const& name); - class StructuredLogAttributes; + [[nodiscard]] static JsonLogAttributes + combine(AttributeFields const& a, AttributeFields const& b); + + AttributeFields& + contextValues() + { + return contextValues_; + } + + [[nodiscard]] AttributeFields const& + contextValues() const + { + return contextValues_; + } + + rapidjson::MemoryPoolAllocator<>& + allocator() + { + return allocator_; + } + + private: + AttributeFields contextValues_; + rapidjson::MemoryPoolAllocator<> allocator_; + + friend class Journal; + }; + + struct JsonLogContext + { + std::source_location location = {}; + rapidjson::Value messageParams; + rapidjson::MemoryPoolAllocator<> allocator; + + JsonLogContext() = default; + + void + reset(std::source_location location_) noexcept + { + location = location_; + messageParams = rapidjson::Value{}; + messageParams.SetObject(); + allocator.Clear(); + } + }; private: // Severity level / threshold of a Journal message. using Severity = severities::Severity; - std::unique_ptr m_attributes; + std::optional m_attributes; + static std::optional globalLogAttributes_; + static std::mutex globalLogAttributesMutex_; + static bool m_jsonLogsEnabled; + + static thread_local JsonLogContext currentJsonLogContext_; - static std::unique_ptr m_structuredJournalImpl; // Invariant: m_sink always points to a valid Sink Sink* m_sink = nullptr; + static void + initMessageContext(std::source_location location); + + static std::string + formatLog(std::string const& message, + severities::Severity severity, + std::optional const& attributes = std::nullopt); public: //-------------------------------------------------------------------------- static void - enableStructuredJournal(std::unique_ptr impl) - { - m_structuredJournalImpl = std::move(impl); - } + enableStructuredJournal(); static void - disableStructuredJournal() - { - m_structuredJournalImpl = nullptr; - } + disableStructuredJournal(); static bool - isStructuredJournalEnabled() - { - return m_structuredJournalImpl != nullptr; - } - - class StructuredJournalImpl - { - public: - StructuredJournalImpl() = default; - StructuredJournalImpl(StructuredJournalImpl const&) = default; - virtual void - initMessageContext(std::source_location location) = 0; - virtual void - flush( - Sink* sink, - severities::Severity level, - std::string const& text, - StructuredLogAttributes* attributes) = 0; - virtual ~StructuredJournalImpl() = default; - }; - - class StructuredLogAttributes - { - public: - StructuredLogAttributes() = default; - StructuredLogAttributes(StructuredLogAttributes const&) = default; - virtual void - setModuleName(std::string const& name) = 0; - virtual std::unique_ptr - clone() const = 0; - virtual void - combine(std::unique_ptr const& attributes) = 0; - virtual void - combine(std::unique_ptr&& attributes) = 0; - virtual ~StructuredLogAttributes() = default; - }; + isStructuredJournalEnabled(); /** Abstraction for the underlying message destination. */ class Sink @@ -214,25 +301,25 @@ class Journal public: ScopedStream(ScopedStream const& other) : ScopedStream( - other.m_attributes ? other.m_attributes->clone() : nullptr, + other.m_attributes, other.m_sink, other.m_level) { } ScopedStream( - std::unique_ptr attributes, + std::optional attributes, Sink& sink, Severity level); template ScopedStream( - std::unique_ptr attributes, + std::optional attributes, Stream const& stream, T const& t); ScopedStream( - std::unique_ptr attributes, + std::optional attributes, Stream const& stream, std::ostream& manip(std::ostream&)); @@ -255,7 +342,7 @@ class Journal operator<<(T const& t) const; private: - std::unique_ptr m_attributes; + std::optional m_attributes; Sink& m_sink; Severity const m_level; std::ostringstream mutable m_ostream; @@ -291,7 +378,7 @@ class Journal Constructor is inlined so checking active() very inexpensive. */ Stream( - std::unique_ptr attributes, + std::optional attributes, Sink& sink, Severity level) : m_attributes(std::move(attributes)), m_sink(sink), m_level(level) @@ -304,7 +391,7 @@ class Journal /** Construct or copy another Stream. */ Stream(Stream const& other) : Stream( - other.m_attributes ? other.m_attributes->clone() : nullptr, + other.m_attributes, other.m_sink, other.m_level) { @@ -353,7 +440,7 @@ class Journal /** @} */ private: - std::unique_ptr m_attributes; + std::optional m_attributes; Sink& m_sink; Severity m_level; }; @@ -372,47 +459,29 @@ class Journal /** Journal has no default constructor. */ Journal() = delete; - Journal(Journal const& other) : Journal(other, nullptr) - { - } - Journal( Journal const& other, - std::unique_ptr attributes) - : m_sink(other.m_sink) - { - if (attributes) - m_attributes = std::move(attributes); - if (other.m_attributes) - { - if (m_attributes) - m_attributes->combine(other.m_attributes); - else - m_attributes = other.m_attributes->clone(); - } - } - - Journal( - Journal&& other, - std::unique_ptr attributes = {}) noexcept + std::optional attributes = std::nullopt) : m_sink(other.m_sink) { - if (attributes) - m_attributes = std::move(attributes); - if (other.m_attributes) + if (attributes.has_value()) + m_attributes = std::move(attributes.value()); + if (other.m_attributes.has_value()) { - if (m_attributes) - m_attributes->combine(std::move(other.m_attributes)); + if (m_attributes.has_value()) + m_attributes = JsonLogAttributes::combine( + other.m_attributes->contextValues_, + m_attributes->contextValues_ + ); else - m_attributes = std::move(other.m_attributes); + m_attributes = other.m_attributes; } } - /** Create a journal that writes to the specified sink. */ - Journal( + explicit Journal( Sink& sink, std::string const& name = {}, - std::unique_ptr attributes = {}) + std::optional attributes = std::nullopt) : m_sink(&sink) { if (attributes) @@ -425,9 +494,11 @@ class Journal Journal& operator=(Journal const& other) { + if (&other == this) + return *this; + m_sink = other.m_sink; - if (other.m_attributes) - m_attributes = other.m_attributes->clone(); + m_attributes = other.m_attributes; return *this; } @@ -435,8 +506,7 @@ class Journal operator=(Journal&& other) noexcept { m_sink = other.m_sink; - if (other.m_attributes) - m_attributes = std::move(other.m_attributes); + m_attributes = std::move(other.m_attributes); return *this; } @@ -452,7 +522,7 @@ class Journal stream(Severity level) const { return Stream( - m_attributes ? m_attributes->clone() : nullptr, *m_sink, level); + m_attributes, *m_sink, level); } /** Returns `true` if any message would be logged at this severity level. @@ -470,10 +540,10 @@ class Journal Stream trace(std::source_location location = std::source_location::current()) const { - if (m_structuredJournalImpl) - m_structuredJournalImpl->initMessageContext(location); + if (m_jsonLogsEnabled) + initMessageContext(location); return { - m_attributes ? m_attributes->clone() : nullptr, + m_attributes, *m_sink, severities::kTrace}; } @@ -481,10 +551,10 @@ class Journal Stream debug(std::source_location location = std::source_location::current()) const { - if (m_structuredJournalImpl) - m_structuredJournalImpl->initMessageContext(location); + if (m_jsonLogsEnabled) + initMessageContext(location); return { - m_attributes ? m_attributes->clone() : nullptr, + m_attributes, *m_sink, severities::kDebug}; } @@ -492,10 +562,10 @@ class Journal Stream info(std::source_location location = std::source_location::current()) const { - if (m_structuredJournalImpl) - m_structuredJournalImpl->initMessageContext(location); + if (m_jsonLogsEnabled) + initMessageContext(location); return { - m_attributes ? m_attributes->clone() : nullptr, + m_attributes, *m_sink, severities::kInfo}; } @@ -503,10 +573,12 @@ class Journal Stream warn(std::source_location location = std::source_location::current()) const { - if (m_structuredJournalImpl) - m_structuredJournalImpl->initMessageContext(location); + const char* a = "a"; + rapidjson::Value v{a, 1}; + if (m_jsonLogsEnabled) + initMessageContext(location); return { - m_attributes ? m_attributes->clone() : nullptr, + m_attributes, *m_sink, severities::kWarning}; } @@ -514,10 +586,10 @@ class Journal Stream error(std::source_location location = std::source_location::current()) const { - if (m_structuredJournalImpl) - m_structuredJournalImpl->initMessageContext(location); + if (m_jsonLogsEnabled) + initMessageContext(location); return { - m_attributes ? m_attributes->clone() : nullptr, + m_attributes, *m_sink, severities::kError}; } @@ -525,14 +597,25 @@ class Journal Stream fatal(std::source_location location = std::source_location::current()) const { - if (m_structuredJournalImpl) - m_structuredJournalImpl->initMessageContext(location); + if (m_jsonLogsEnabled) + initMessageContext(location); return { - m_attributes ? m_attributes->clone() : nullptr, + m_attributes, *m_sink, severities::kFatal}; } /** @} */ + + static void + addGlobalAttributes(JsonLogAttributes globalLogAttributes) + { + std::lock_guard lock(globalLogAttributesMutex_); + if (!globalLogAttributes_) + { + globalLogAttributes_ = JsonLogAttributes{}; + } + globalLogAttributes_ = JsonLogAttributes::combine(globalLogAttributes_->contextValues(), globalLogAttributes.contextValues()); + } }; #ifndef __INTELLISENSE__ @@ -548,7 +631,7 @@ static_assert(std::is_nothrow_destructible::value == true, ""); template Journal::ScopedStream::ScopedStream( - std::unique_ptr attributes, + std::optional attributes, Stream const& stream, T const& t) : ScopedStream(std::move(attributes), stream.sink(), stream.level()) @@ -570,7 +653,7 @@ template Journal::ScopedStream Journal::Stream::operator<<(T const& t) const { - return {m_attributes ? m_attributes->clone() : nullptr, *this, t}; + return {m_attributes, *this, t}; } namespace detail { @@ -642,4 +725,127 @@ using logwstream = basic_logstream; } // namespace beast + +namespace ripple::log { + +namespace detail { +template +void setJsonValue( + rapidjson::Value& object, + rapidjson::MemoryPoolAllocator<>& allocator, + char const* name, + T&& value, + std::ostream* outStream) +{ + using ValueType = std::decay_t; + rapidjson::Value jsonValue; + if constexpr (std::constructible_from&>) + { + jsonValue = rapidjson::Value{value, allocator}; + if (outStream) + { + (*outStream) << value; + } + } + else if constexpr (std::constructible_from) + { + jsonValue = rapidjson::Value{value}; + if (outStream) + { + (*outStream) << value; + } + } + else if constexpr (std::same_as) + { + jsonValue = rapidjson::Value{value.c_str(), allocator}; + if (outStream) + { + (*outStream) << value; + } + } + else + { + std::ostringstream oss; + oss << value; + + jsonValue = rapidjson::Value{oss.str().c_str(), allocator}; + + if (outStream) + { + (*outStream) << oss.str(); + } + } + + object.AddMember( + rapidjson::StringRef(name), + std::move(jsonValue), + allocator + ); +} +} + +template +std::ostream& +operator<<(std::ostream& os, LogParameter const& param) +{ + if (!beast::Journal::m_jsonLogsEnabled) + return os; + detail::setJsonValue( + beast::Journal::currentJsonLogContext_.messageParams, + beast::Journal::currentJsonLogContext_.allocator, + param.name_, param.value_, &os); + return os; +} + +template +std::ostream& +operator<<(std::ostream& os, LogField const& param) +{ + if (!beast::Journal::m_jsonLogsEnabled) + return os; + detail::setJsonValue( + beast::Journal::currentJsonLogContext_.messageParams, + beast::Journal::currentJsonLogContext_.allocator, + param.name_, param.value_, nullptr); + return os; +} + +template +LogParameter +param(char const* name, T&& value) +{ + return LogParameter{name, std::forward(value)}; +} + +template +LogField +field(char const* name, T&& value) +{ + return LogField{name, std::forward(value)}; +} + +template +[[nodiscard]] beast::Journal::JsonLogAttributes +attributes(Pair&&... pairs) +{ + beast::Journal::JsonLogAttributes result; + + (detail::setJsonValue( + result.contextValues(), + result.allocator(), + pairs.first, + pairs.second, nullptr), ...); + + return result; +} + +template +[[nodiscard]] std::pair> +attr(char const* name, T&& value) +{ + return std::make_pair(name, std::forward(value)); +} + +} + #endif diff --git a/include/xrpl/resource/detail/Logic.h b/include/xrpl/resource/detail/Logic.h index b8288294fe9..8bfa2c2ac83 100644 --- a/include/xrpl/resource/detail/Logic.h +++ b/include/xrpl/resource/detail/Logic.h @@ -32,7 +32,6 @@ #include #include #include -#include #include diff --git a/include/xrpl/server/detail/BasePeer.h b/include/xrpl/server/detail/BasePeer.h index bdae5046198..0a578fc65ea 100644 --- a/include/xrpl/server/detail/BasePeer.h +++ b/include/xrpl/server/detail/BasePeer.h @@ -25,7 +25,6 @@ #include #include #include -#include #include @@ -86,11 +85,11 @@ BasePeer::BasePeer( , remote_address_(remote_address) , j_(journal, log::attributes( - {{"PeerID", + log::attr("PeerID", [] { static std::atomic id{0}; return "##" + std::to_string(++id) + " "; - }()}})) + }()))) , work_(boost::asio::make_work_guard(executor)) , strand_(boost::asio::make_strand(executor)) { diff --git a/include/xrpl/telemetry/JsonLogs.h b/include/xrpl/telemetry/JsonLogs.h deleted file mode 100644 index 1ed3e16a8dc..00000000000 --- a/include/xrpl/telemetry/JsonLogs.h +++ /dev/null @@ -1,220 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2025 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_LOGGING_STRUCTUREDJOURNAL_H_INCLUDED -#define RIPPLE_LOGGING_STRUCTUREDJOURNAL_H_INCLUDED - -#include -#include - -#include -#include -#include -#include - -namespace ripple::log { - -template -class LogParameter -{ -public: - template - LogParameter(char const* name, TArg&& value) - : name_(name), value_(std::forward(value)) - { - } - -private: - char const* name_; - T value_; - - template - friend std::ostream& - operator<<(std::ostream& os, LogParameter const&); -}; - -template -class LogField -{ -public: - template - LogField(char const* name, TArg&& value) - : name_(name), value_(std::forward(value)) - { - } - -private: - char const* name_; - T value_; - - template - friend std::ostream& - operator<<(std::ostream& os, LogField const&); -}; - -class JsonLogAttributes : public beast::Journal::StructuredLogAttributes -{ -public: - using AttributeFields = std::unordered_map; - using Pair = AttributeFields::value_type; - - explicit JsonLogAttributes(AttributeFields contextValues = {}); - - void - setModuleName(std::string const& name) override; - - [[nodiscard]] std::unique_ptr - clone() const override; - - void - combine(std::unique_ptr const& context) override; - - void - combine(std::unique_ptr&& context) override; - - AttributeFields& - contextValues() - { - return contextValues_; - } - -private: - AttributeFields contextValues_; -}; - -class JsonStructuredJournal : public beast::Journal::StructuredJournalImpl -{ -private: - struct Logger - { - std::source_location location = {}; - Json::Value messageParams; - - Logger() = default; - Logger( - JsonStructuredJournal const* journal, - std::source_location location); - - void - write( - beast::Journal::Sink* sink, - beast::severities::Severity level, - std::string const& text, - beast::Journal::StructuredLogAttributes* context) const; - }; - - [[nodiscard]] Logger - logger(std::source_location location) const; - - static thread_local Logger currentLogger_; - - template - friend std::ostream& - operator<<(std::ostream& os, LogParameter const&); - - template - friend std::ostream& - operator<<(std::ostream& os, LogField const&); - -public: - void - initMessageContext(std::source_location location) override; - - void - flush( - beast::Journal::Sink* sink, - beast::severities::Severity level, - std::string const& text, - beast::Journal::StructuredLogAttributes* context) override; -}; - -template -std::ostream& -operator<<(std::ostream& os, LogParameter const& param) -{ - using ValueType = std::decay_t; - // TODO: Update the Json library to support 64-bit integer values. - if constexpr ( - std::constructible_from && - (!std::is_integral_v || - sizeof(ValueType) <= sizeof(Json::Int))) - { - JsonStructuredJournal::currentLogger_.messageParams[param.name_] = - Json::Value{param.value_}; - return os << param.value_; - } - else - { - std::ostringstream oss; - oss << param.value_; - - JsonStructuredJournal::currentLogger_.messageParams[param.name_] = - oss.str(); - return os << oss.str(); - } -} - -template -std::ostream& -operator<<(std::ostream& os, LogField const& param) -{ - using ValueType = std::decay_t; - // TODO: Update the Json library to support 64-bit integer values. - if constexpr ( - std::constructible_from && - (!std::is_integral_v || - sizeof(ValueType) <= sizeof(Json::Int))) - { - JsonStructuredJournal::currentLogger_.messageParams[param.name_] = - Json::Value{param.value_}; - } - else - { - std::ostringstream oss; - oss << param.value_; - - JsonStructuredJournal::currentLogger_.messageParams[param.name_] = - oss.str(); - } - return os; -} - -template -LogParameter -param(char const* name, T&& value) -{ - return LogParameter{name, std::forward(value)}; -} - -template -LogField -field(char const* name, T&& value) -{ - return LogField{name, std::forward(value)}; -} - -[[nodiscard]] inline std::unique_ptr -attributes(std::initializer_list const& fields) -{ - return std::make_unique(fields); -} - -} // namespace ripple::log - -#endif diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index e7a929af89f..0ddd56f17d2 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -38,9 +38,6 @@ namespace ripple { -std::unique_ptr - Logs::globalLogAttributes_; - Logs::Sink::Sink( std::string const& partition, beast::severities::Severity thresh, @@ -162,16 +159,9 @@ Logs::operator[](std::string const& name) beast::Journal Logs::journal( std::string const& name, - std::unique_ptr attributes) + std::optional attributes) { - if (globalLogAttributes_) - { - if (attributes) - attributes->combine(globalLogAttributes_); - else - attributes = globalLogAttributes_->clone(); - } - return beast::Journal(get(name), name, std::move(attributes)); + return beast::Journal{get(name), name, std::move(attributes)}; } beast::severities::Severity diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index d43c05f0214..bcb9c3048e9 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -19,13 +19,22 @@ #include +#include +#include +#include + +#include +#include #include #include #include namespace beast { -std::unique_ptr Journal::m_structuredJournalImpl; +std::optional Journal::globalLogAttributes_; +std::mutex Journal::globalLogAttributesMutex_; +bool Journal::m_jsonLogsEnabled = false; +thread_local Journal::JsonLogContext Journal::currentJsonLogContext_{}; //------------------------------------------------------------------------------ @@ -113,6 +122,184 @@ severities::to_string(Severity severity) } return ""; } + +Journal::JsonLogAttributes::JsonLogAttributes() +{ + contextValues_.SetObject(); +} + +Journal::JsonLogAttributes::JsonLogAttributes(JsonLogAttributes const& other) +{ + contextValues_.SetObject(); + contextValues_.CopyFrom(other.contextValues_, allocator_); +} + +Journal::JsonLogAttributes& +Journal::JsonLogAttributes::operator=(JsonLogAttributes const& other) +{ + if (&other == this) + { + return *this; + } + contextValues_.CopyFrom(other.contextValues_, allocator_); + return *this; +} + +void +Journal::JsonLogAttributes::setModuleName(std::string const& name) +{ + contextValues_.AddMember( + rapidjson::StringRef("Module"), + rapidjson::Value{name.c_str(), allocator_}, + allocator_ + ); +} + +Journal::JsonLogAttributes +Journal::JsonLogAttributes::combine( + AttributeFields const& a, + AttributeFields const& b) +{ + JsonLogAttributes result; + + result.contextValues_.CopyFrom(a, result.allocator_); + + for (auto& member : b.GetObject()) + { + auto val = rapidjson::Value{ member.value, result.allocator_ }; + if (result.contextValues_.HasMember(member.name)) + { + result.contextValues_[member.name] = std::move(val); + } + else + { + result.contextValues_.AddMember( + rapidjson::Value{member.name, result.allocator_}, + std::move(val), + result.allocator_); + } + } + + return result; +} + +void +Journal::initMessageContext(std::source_location location) +{ + currentJsonLogContext_.reset(location); +} + +std::string +Journal::formatLog( + std::string const& message, + severities::Severity severity, + std::optional const& attributes) +{ + if (!m_jsonLogsEnabled) + { + return message; + } + + rapidjson::Document doc{¤tJsonLogContext_.allocator}; + rapidjson::Value logContext; + logContext.SetObject(); + + if (globalLogAttributes_) + { + for (auto const& [key, value] : globalLogAttributes_->contextValues().GetObject()) + { + rapidjson::Value jsonValue; + jsonValue.CopyFrom(value, currentJsonLogContext_.allocator); + + logContext.AddMember( + rapidjson::Value{key, currentJsonLogContext_.allocator}, + std::move(jsonValue), + currentJsonLogContext_.allocator); + } + } + + if (attributes.has_value()) + { + for (auto const& [key, value] : attributes->contextValues().GetObject()) + { + rapidjson::Value jsonValue; + jsonValue.CopyFrom(value, currentJsonLogContext_.allocator); + + logContext.AddMember( + rapidjson::Value{key, currentJsonLogContext_.allocator}, + std::move(jsonValue), + currentJsonLogContext_.allocator); + } + } + logContext.AddMember( + rapidjson::StringRef("Function"), + rapidjson::StringRef(currentJsonLogContext_.location.function_name()), + currentJsonLogContext_.allocator); + + logContext.AddMember( + rapidjson::StringRef("File"), + rapidjson::StringRef(currentJsonLogContext_.location.file_name()), + currentJsonLogContext_.allocator); + + logContext.AddMember( + rapidjson::StringRef("Line"), + currentJsonLogContext_.location.line(), + currentJsonLogContext_.allocator); + std::stringstream threadIdStream; + threadIdStream << std::this_thread::get_id(); + auto threadIdStr = threadIdStream.str(); + logContext.AddMember( + rapidjson::StringRef("ThreadId"), + rapidjson::StringRef(threadIdStr.c_str()), + currentJsonLogContext_.allocator); + logContext.AddMember( + rapidjson::StringRef("Params"), + std::move(currentJsonLogContext_.messageParams), + currentJsonLogContext_.allocator); + currentJsonLogContext_.messageParams = rapidjson::Value{}; + currentJsonLogContext_.messageParams.SetObject(); + auto severityStr = to_string(severity); + logContext.AddMember( + rapidjson::StringRef("Level"), + rapidjson::StringRef(severityStr.c_str()), + currentJsonLogContext_.allocator); + logContext.AddMember( + rapidjson::StringRef("Message"), + rapidjson::StringRef(message.c_str()), + currentJsonLogContext_.allocator); + logContext.AddMember( + rapidjson::StringRef("Time"), + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(), + currentJsonLogContext_.allocator); + + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + + logContext.Accept(writer); + + return {buffer.GetString()}; +} + +void +Journal::enableStructuredJournal() +{ + m_jsonLogsEnabled = true; +} + +void +Journal::disableStructuredJournal() +{ + m_jsonLogsEnabled = false; +} + +bool +Journal::isStructuredJournalEnabled() +{ + return m_jsonLogsEnabled; +} + Journal::Sink::Sink(Severity thresh, bool console) : thresh_(thresh), m_console(console) { @@ -153,7 +340,7 @@ Journal::Sink::threshold(Severity thresh) //------------------------------------------------------------------------------ Journal::ScopedStream::ScopedStream( - std::unique_ptr attributes, + std::optional attributes, Sink& sink, Severity level) : m_attributes(std::move(attributes)), m_sink(sink), m_level(level) @@ -163,7 +350,7 @@ Journal::ScopedStream::ScopedStream( } Journal::ScopedStream::ScopedStream( - std::unique_ptr attributes, + std::optional attributes, Stream const& stream, std::ostream& manip(std::ostream&)) : ScopedStream(std::move(attributes), stream.sink(), stream.level()) @@ -177,21 +364,9 @@ Journal::ScopedStream::~ScopedStream() if (!s.empty()) { if (s == "\n") - { - if (m_structuredJournalImpl) - m_structuredJournalImpl->flush( - &m_sink, m_level, "", m_attributes.get()); - else - m_sink.write(m_level, ""); - } + m_sink.write(m_level, formatLog("", m_level, m_attributes)); else - { - if (m_structuredJournalImpl) - m_structuredJournalImpl->flush( - &m_sink, m_level, s, m_attributes.get()); - else - m_sink.write(m_level, s); - } + m_sink.write(m_level, formatLog(s, m_level, m_attributes)); } } @@ -206,7 +381,7 @@ Journal::ScopedStream::operator<<(std::ostream& manip(std::ostream&)) const Journal::ScopedStream Journal::Stream::operator<<(std::ostream& manip(std::ostream&)) const { - return {m_attributes ? m_attributes->clone() : nullptr, *this, manip}; + return {m_attributes, *this, manip}; } } // namespace beast diff --git a/src/libxrpl/telemetry/JsonLogs.cpp b/src/libxrpl/telemetry/JsonLogs.cpp deleted file mode 100644 index 1006c8becc6..00000000000 --- a/src/libxrpl/telemetry/JsonLogs.cpp +++ /dev/null @@ -1,129 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2025 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include - -#include -#include - -namespace ripple::log { - -thread_local JsonStructuredJournal::Logger - JsonStructuredJournal::currentLogger_{}; - -JsonLogAttributes::JsonLogAttributes(AttributeFields contextValues) - : contextValues_(std::move(contextValues)) -{ -} - -void -JsonLogAttributes::setModuleName(std::string const& name) -{ - contextValues()["Module"] = name; -} - -std::unique_ptr -JsonLogAttributes::clone() const -{ - return std::make_unique(*this); -} - -void -JsonLogAttributes::combine( - std::unique_ptr const& context) -{ - auto structuredContext = - static_cast(context.get()); - contextValues_.merge(AttributeFields{structuredContext->contextValues_}); -} - -void -JsonLogAttributes::combine(std::unique_ptr&& context) -{ - auto structuredContext = static_cast(context.get()); - - if (contextValues_.empty()) - contextValues_ = std::move(structuredContext->contextValues_); - else - contextValues_.merge(structuredContext->contextValues_); -} - -JsonStructuredJournal::Logger::Logger( - JsonStructuredJournal const* journal, - std::source_location location) - : location(location) -{ -} - -void -JsonStructuredJournal::Logger::write( - beast::Journal::Sink* sink, - beast::severities::Severity level, - std::string const& text, - beast::Journal::StructuredLogAttributes* context) const -{ - Json::Value globalContext; - if (context) - { - auto jsonContext = static_cast(context); - for (auto const& [key, value] : jsonContext->contextValues()) - globalContext[key] = value; - } - globalContext["Function"] = location.function_name(); - globalContext["File"] = location.file_name(); - globalContext["Line"] = location.line(); - std::stringstream threadIdStream; - threadIdStream << std::this_thread::get_id(); - globalContext["ThreadId"] = threadIdStream.str(); - globalContext["Params"] = messageParams; - globalContext["Level"] = to_string(level); - globalContext["Message"] = text; - globalContext["Time"] = - to_string(std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count()); - - sink->write(level, to_string(globalContext)); -} - -JsonStructuredJournal::Logger -JsonStructuredJournal::logger(std::source_location location) const -{ - return Logger{this, location}; -} - -void -JsonStructuredJournal::initMessageContext(std::source_location location) -{ - currentLogger_ = logger(location); -} - -void -JsonStructuredJournal::flush( - beast::Journal::Sink* sink, - beast::severities::Severity level, - std::string const& text, - beast::Journal::StructuredLogAttributes* context) -{ - currentLogger_.write(sink, level, text, context); -} - -} // namespace ripple::log diff --git a/src/test/core/Coroutine_test.cpp b/src/test/core/Coroutine_test.cpp index 8458da647d4..bbdd2e9b001 100644 --- a/src/test/core/Coroutine_test.cpp +++ b/src/test/core/Coroutine_test.cpp @@ -175,12 +175,72 @@ class Coroutine_test : public beast::unit_test::suite BEAST_EXPECT(*lv == -1); } + void + test_yield_and_stop() + { + + using namespace std::chrono_literals; + using namespace jtx; + + testcase("yield and stop"); + + Env env(*this, envconfig([](std::unique_ptr cfg) { + cfg->FORCE_MULTI_THREAD = true; + return cfg; + })); + + std::shared_ptr c; + std::mutex mutexStop; + std::mutex mutexYield; + std::condition_variable cond; + std::condition_variable condYield; + bool yielded = false; + bool stopped = false; + + env.app().getJobQueue().postCoro( + jtCLIENT, "Coroutine-Test", [&](auto const& cr) { + c = cr; + { + std::unique_lock lock(mutexYield); + yielded = true; + condYield.notify_all(); + } + c->yield(); + // Just to keep this job alive + std::this_thread::sleep_for(5ms); + }); + std::thread th{[&]() { + std::unique_lock lock(mutexStop); + cond.wait(lock, [&]() { return stopped; }); + // Delay a bit to wait for stop() to be called + std::this_thread::sleep_for(1ms); + c->post(); + }}; + + // Delay a bit to wait for yield() to be called + std::this_thread::sleep_for(1ms); + std::unique_lock lockYield(mutexYield); + condYield.wait(lockYield, [&]() { return yielded; }); + { + std::unique_lock lock(mutexStop); + stopped = true; + cond.notify_all(); + } + env.app().getJobQueue().stop(); + try + { + th.join(); + } catch (const std::exception& e) {} + pass(); + } + void run() override { - correct_order(); - incorrect_order(); - thread_specific_storage(); + // correct_order(); + // incorrect_order(); + // thread_specific_storage(); + test_yield_and_stop(); } }; diff --git a/src/test/csf/Peer.h b/src/test/csf/Peer.h index 783da4f2494..6785846b0f4 100644 --- a/src/test/csf/Peer.h +++ b/src/test/csf/Peer.h @@ -33,7 +33,6 @@ #include #include -#include #include #include @@ -284,7 +283,7 @@ struct Peer TrustGraph& tg, CollectorRefs& c, beast::Journal jIn) - : j(jIn, log::attributes({{"Peer", "Peer " + to_string(i)}})) + : j(jIn, log::attributes(log::attr("Peer", "Peer " + to_string(i)))) , consensus(s.clock(), *this, j) , id{i} , key{id, 0} diff --git a/src/tests/libxrpl/CMakeLists.txt b/src/tests/libxrpl/CMakeLists.txt index 73520a51282..72f0e532ce8 100644 --- a/src/tests/libxrpl/CMakeLists.txt +++ b/src/tests/libxrpl/CMakeLists.txt @@ -2,10 +2,11 @@ include(xrpl_add_test) # Test requirements. find_package(doctest REQUIRED) +find_package(RapidJSON REQUIRED) # Common library dependencies for the rest of the tests. add_library(xrpl.imports.test INTERFACE) -target_link_libraries(xrpl.imports.test INTERFACE doctest::doctest xrpl.libxrpl) +target_link_libraries(xrpl.imports.test INTERFACE doctest::doctest rapidjson xrpl.libxrpl) # One test for each module. xrpl_add_test(basics) diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index 047bf337322..5d6e7ca2cca 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -18,9 +18,8 @@ //============================================================================== #include -#include -#include +#include #include using namespace ripple; @@ -108,8 +107,7 @@ TEST_CASE("Test format output") TEST_CASE("Test format output when structured logs are enabled") { - auto structuredJournal = std::make_unique(); - beast::Journal::enableStructuredJournal(std::move(structuredJournal)); + beast::Journal::enableStructuredJournal(); std::string output; Logs::format(output, "Message", beast::severities::kDebug, "Test"); @@ -121,8 +119,6 @@ TEST_CASE("Test format output when structured logs are enabled") TEST_CASE("Enable json logs") { - auto structuredJournal = std::make_unique(); - std::stringstream logStream; MockLogs logs{logStream, beast::severities::kAll}; @@ -133,78 +129,429 @@ TEST_CASE("Enable json logs") logStream.str(""); - beast::Journal::enableStructuredJournal(std::move(structuredJournal)); + beast::Journal::enableStructuredJournal(); logs.journal("Test").debug() << "\n"; - Json::Reader reader; - Json::Value jsonLog; - bool result = reader.parse(logStream.str(), jsonLog); + rapidjson::Document doc; + doc.Parse(logStream.str().c_str()); - CHECK(result); + CHECK(doc.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); - CHECK(jsonLog.isObject()); - CHECK(jsonLog.isMember("Message")); - CHECK(jsonLog["Message"].isString()); - CHECK(jsonLog["Message"].asString() == ""); + CHECK(doc.IsObject()); + CHECK(doc.HasMember("Message")); + CHECK(doc["Message"].IsString()); + CHECK(doc["Message"].GetString() == std::string{""}); beast::Journal::disableStructuredJournal(); } TEST_CASE("Global attributes") { - auto structuredJournal = std::make_unique(); - std::stringstream logStream; MockLogs logs{logStream, beast::severities::kAll}; - beast::Journal::enableStructuredJournal(std::move(structuredJournal)); - MockLogs::setGlobalAttributes(log::attributes({{"Field1", "Value1"}})); + beast::Journal::enableStructuredJournal(); + beast::Journal::addGlobalAttributes(log::attributes(log::attr("Field1", "Value1"))); logs.journal("Test").debug() << "Test"; - Json::Reader reader; - Json::Value jsonLog; - bool result = reader.parse(logStream.str(), jsonLog); + rapidjson::Document jsonLog; + jsonLog.Parse(logStream.str().c_str()); - CHECK(result); + CHECK(jsonLog.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); - CHECK(jsonLog.isObject()); - CHECK(jsonLog.isMember("Field1")); - CHECK(jsonLog["Field1"].isString()); - CHECK(jsonLog["Field1"].asString() == "Value1"); + CHECK(jsonLog.IsObject()); + CHECK(jsonLog.HasMember("Field1")); + CHECK(jsonLog["Field1"].IsString()); + CHECK(jsonLog["Field1"].GetString() == std::string{"Value1"}); beast::Journal::disableStructuredJournal(); } TEST_CASE("Global attributes inheritable") { - auto structuredJournal = std::make_unique(); - std::stringstream logStream; MockLogs logs{logStream, beast::severities::kAll}; - beast::Journal::enableStructuredJournal(std::move(structuredJournal)); - MockLogs::setGlobalAttributes(log::attributes({{"Field1", "Value1"}})); + beast::Journal::enableStructuredJournal(); + beast::Journal::addGlobalAttributes(log::attributes(log::attr("Field1", "Value1"))); logs.journal( "Test", - log::attributes({{"Field1", "Value3"}, {"Field2", "Value2"}})) + log::attributes(log::attr("Field1", "Value3"), log::attr("Field2", "Value2"))) .debug() << "Test"; - Json::Reader reader; - Json::Value jsonLog; - bool result = reader.parse(logStream.str(), jsonLog); + rapidjson::Document jsonLog; + jsonLog.Parse(logStream.str().c_str()); - CHECK(result); + CHECK(jsonLog.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); - CHECK(jsonLog.isObject()); - CHECK(jsonLog.isMember("Field1")); - CHECK(jsonLog["Field1"].isString()); + CHECK(jsonLog.IsObject()); + CHECK(jsonLog.HasMember("Field1")); + CHECK(jsonLog["Field1"].IsString()); // Field1 should be overwritten to Value3 - CHECK(jsonLog["Field1"].asString() == "Value3"); - CHECK(jsonLog["Field2"].isString()); - CHECK(jsonLog["Field2"].asString() == "Value2"); + CHECK(jsonLog["Field1"].GetString() == std::string{"Value3"}); + CHECK(jsonLog["Field2"].IsString()); + CHECK(jsonLog["Field2"].GetString() == std::string{"Value2"}); beast::Journal::disableStructuredJournal(); +} + +/** + * @brief sink for writing all log messages to a stringstream + */ +class MockSink : public beast::Journal::Sink +{ + std::stringstream& strm_; + +public: + MockSink(beast::severities::Severity threshold, std::stringstream& strm) + : beast::Journal::Sink(threshold, false), strm_(strm) + { + } + + void + write(beast::severities::Severity level, std::string const& text) override + { + strm_ << text; + } + + void + writeAlways(beast::severities::Severity level, std::string const& text) + override + { + strm_ << text; + } +}; + +class JsonLogStreamFixture +{ +public: + JsonLogStreamFixture() + : sink_(beast::severities::kAll, logStream_), j_(sink_) + { + beast::Journal::enableStructuredJournal(); + } + + ~JsonLogStreamFixture() + { + beast::Journal::disableStructuredJournal(); + } + + std::stringstream& + stream() + { + return logStream_; + } + + beast::Journal& + journal() + { + return j_; + } + +private: + MockSink sink_; + std::stringstream logStream_; + beast::Journal j_; +}; + +TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") +{ + journal().debug() << std::boolalpha << true << std::noboolalpha << " Test " + << std::boolalpha << false; + + rapidjson::Document logValue; + logValue.Parse(stream().str().c_str()); + + CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + + CHECK(logValue.IsObject()); + CHECK(logValue.HasMember("Function")); + CHECK(logValue.HasMember("File")); + CHECK(logValue.HasMember("Line")); + CHECK(logValue.HasMember("ThreadId")); + CHECK(logValue.HasMember("Params")); + CHECK(logValue.HasMember("Level")); + CHECK(logValue.HasMember("Message")); + CHECK(logValue.HasMember("Time")); + + CHECK(logValue["Function"].IsString()); + CHECK(logValue["File"].IsString()); + CHECK(logValue["Line"].IsNumber()); + CHECK(logValue["Params"].IsObject()); + CHECK(logValue["Message"].IsString()); + CHECK(logValue["Message"].GetString() == std::string{"true Test false"}); +} + +TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") +{ + { + stream().str(""); + journal().trace() << "Test"; + + rapidjson::Document logValue; + logValue.Parse(stream().str().c_str()); + + CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + + CHECK( + logValue["Level"].GetString() == + beast::severities::to_string(beast::severities::kTrace)); + } + + { + stream().str(""); + journal().debug() << "Test"; + + rapidjson::Document logValue; + logValue.Parse(stream().str().c_str()); + + CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + + + CHECK( + logValue["Level"].GetString() == + beast::severities::to_string(beast::severities::kDebug)); + } + + { + stream().str(""); + journal().info() << "Test"; + + rapidjson::Document logValue; + logValue.Parse(stream().str().c_str()); + + CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + + + CHECK( + logValue["Level"].GetString() == + beast::severities::to_string(beast::severities::kInfo)); + } + + { + stream().str(""); + journal().warn() << "Test"; + + rapidjson::Document logValue; + logValue.Parse(stream().str().c_str()); + + CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + + + CHECK( + logValue["Level"].GetString() == + beast::severities::to_string(beast::severities::kWarning)); + } + + { + stream().str(""); + journal().error() << "Test"; + + rapidjson::Document logValue; + logValue.Parse(stream().str().c_str()); + + CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + + + CHECK( + logValue["Level"].GetString() == + beast::severities::to_string(beast::severities::kError)); + } + + { + stream().str(""); + journal().fatal() << "Test"; + + rapidjson::Document logValue; + logValue.Parse(stream().str().c_str()); + + CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + + CHECK( + logValue["Level"].GetString() == + beast::severities::to_string(beast::severities::kFatal)); + } +} + +TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogStream") +{ + journal().stream(beast::severities::kError) << "Test"; + + rapidjson::Document logValue; + logValue.Parse(stream().str().c_str()); + + CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + + CHECK( + logValue["Level"].GetString() == + beast::severities::to_string(beast::severities::kError)); +} + +TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogParams") +{ + journal().debug() << "Test: " << log::param("Field1", 1) << ", " + << log::param( + "Field2", + std::numeric_limits::max()); + + rapidjson::Document logValue; + logValue.Parse(stream().str().c_str()); + + CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + + CHECK(logValue["Params"].IsObject()); + CHECK(logValue["Params"]["Field1"].IsNumber()); + CHECK(logValue["Params"]["Field1"].GetInt() == 1); + // UInt64 doesn't fit in Json::Value so it should be converted to a string + // NOTE: We should expect it to be an int64 after we make the json library + // support in64 and uint64 + CHECK(logValue["Params"]["Field2"].IsNumber()); + CHECK(logValue["Params"]["Field2"].GetUint64() == std::numeric_limits::max()); + CHECK(logValue["Message"].IsString()); + CHECK(logValue["Message"].GetString() == std::string{"Test: 1, 18446744073709551615"}); +} + +TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") +{ + journal().debug() << "Test" << log::field("Field1", 1) + << log::field( + "Field2", + std::numeric_limits::max()); + + rapidjson::Document logValue; + logValue.Parse(stream().str().c_str()); + + CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + + + CHECK(logValue["Params"].IsObject()); + CHECK(logValue["Params"]["Field1"].IsNumber()); + CHECK(logValue["Params"]["Field1"].GetInt() == 1); + // UInt64 doesn't fit in Json::Value so it should be converted to a string + // NOTE: We should expect it to be an int64 after we make the json library + // support in64 and uint64 + CHECK(logValue["Params"]["Field2"].IsNumber()); + CHECK(logValue["Params"]["Field2"].GetUint64() == std::numeric_limits::max()); + CHECK(logValue["Message"].IsString()); + CHECK(logValue["Message"].GetString() == std::string{"Test"}); +} + +TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributes") +{ + beast::Journal j{ + journal(), log::attributes(log::attr("Field1", "Value1"), log::attr("Field2", 2))}; + + j.debug() << "Test"; + + rapidjson::Document logValue; + logValue.Parse(stream().str().c_str()); + + CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + + CHECK(logValue["Field1"].IsString()); + CHECK(logValue["Field1"].GetString() == std::string{"Value1"}); + CHECK(logValue["Field2"].IsNumber()); + CHECK(logValue["Field2"].GetInt() == 2); +} + +TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributesInheritable") +{ + beast::Journal j{ + journal(), log::attributes(log::attr("Field1", "Value1"), log::attr("Field2", 2))}; + beast::Journal j2{ + j, log::attributes(log::attr("Field3", "Value3"), log::attr("Field2", 0))}; + + j2.debug() << "Test"; + + rapidjson::Document logValue; + logValue.Parse(stream().str().c_str()); + + CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + + CHECK(logValue["Field1"].IsString()); + CHECK(logValue["Field1"].GetString() == std::string{"Value1"}); + CHECK(logValue["Field3"].IsString()); + CHECK(logValue["Field3"].GetString() == std::string{"Value3"}); + // Field2 should be overwritten to 0 + CHECK(logValue["Field2"].IsNumber()); + CHECK(logValue["Field2"].GetInt() == 0); +} + +TEST_CASE_FIXTURE( + JsonLogStreamFixture, + "TestJournalAttributesInheritableAfterMoving") +{ + beast::Journal j{ + journal(), + log::attributes(log::attr("Field1", "Value1"), log::attr("Field2", 2))}; + beast::Journal j2{ + j, log::attributes(log::attr("Field3", "Value3"), log::attr("Field2", 0))}; + + j2.debug() << "Test"; + + rapidjson::Document logValue; + logValue.Parse(stream().str().c_str()); + + CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + + CHECK(logValue["Field1"].IsString()); + CHECK(logValue["Field1"].GetString() == std::string{"Value1"}); + CHECK(logValue["Field3"].IsString()); + CHECK(logValue["Field3"].GetString() == std::string{"Value3"}); + // Field2 should be overwritten to 0 + CHECK(logValue["Field2"].IsNumber()); + CHECK(logValue["Field2"].GetInt() == 0); +} + +TEST_CASE_FIXTURE( + JsonLogStreamFixture, + "TestJournalAttributesInheritableAfterCopyAssignment") +{ + beast::Journal j{ + std::move(journal()), + log::attributes(log::attr("Field1", "Value1"), log::attr("Field2", 2))}; + + beast::Journal j2{beast::Journal::getNullSink()}; + + j2 = j; + + j2.debug() << "Test"; + + rapidjson::Document logValue; + logValue.Parse(stream().str().c_str()); + + CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + + CHECK(logValue["Field1"].IsString()); + CHECK(logValue["Field1"].GetString() == std::string{"Value1"}); + CHECK(logValue["Field2"].IsNumber()); + CHECK(logValue["Field2"].GetInt() == 2); +} + +TEST_CASE_FIXTURE( + JsonLogStreamFixture, + "TestJournalAttributesInheritableAfterMoveAssignment") +{ + beast::Journal j{ + journal(), + log::attributes(log::attr("Field1", "Value1"), log::attr("Field2", 2))}; + + beast::Journal j2{beast::Journal::getNullSink()}; + + j2 = std::move(j); + + j2.debug() << "Test"; + + rapidjson::Document logValue; + logValue.Parse(stream().str().c_str()); + + CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + + CHECK(logValue["Field1"].IsString()); + CHECK(logValue["Field1"].GetString() == std::string{"Value1"}); + CHECK(logValue["Field2"].IsNumber()); + CHECK(logValue["Field2"].GetInt() == 2); } \ No newline at end of file diff --git a/src/tests/libxrpl/telemetry/json_logs.cpp b/src/tests/libxrpl/telemetry/json_logs.cpp deleted file mode 100644 index 9d95959efe7..00000000000 --- a/src/tests/libxrpl/telemetry/json_logs.cpp +++ /dev/null @@ -1,359 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include - -#include - -using namespace ripple; - -/** - * @brief sink for writing all log messages to a stringstream - */ -class MockSink : public beast::Journal::Sink -{ - std::stringstream& strm_; - -public: - MockSink(beast::severities::Severity threshold, std::stringstream& strm) - : beast::Journal::Sink(threshold, false), strm_(strm) - { - } - - void - write(beast::severities::Severity level, std::string const& text) override - { - strm_ << text; - } - - void - writeAlways(beast::severities::Severity level, std::string const& text) - override - { - strm_ << text; - } -}; - -class JsonLogStreamFixture -{ -public: - JsonLogStreamFixture() - : sink_(beast::severities::kAll, logStream_), j_(sink_) - { - auto structuredJournal = std::make_unique(); - beast::Journal::enableStructuredJournal(std::move(structuredJournal)); - } - - ~JsonLogStreamFixture() - { - beast::Journal::disableStructuredJournal(); - } - - std::stringstream& - stream() - { - return logStream_; - } - - beast::Journal& - journal() - { - return j_; - } - -private: - MockSink sink_; - std::stringstream logStream_; - beast::Journal j_; -}; - -TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") -{ - journal().debug() << std::boolalpha << true << std::noboolalpha << " Test " - << std::boolalpha << false; - - Json::Value logValue; - Json::Reader reader; - reader.parse(stream().str(), logValue); - - CHECK(logValue.isObject()); - CHECK(logValue.isMember("Function")); - CHECK(logValue.isMember("File")); - CHECK(logValue.isMember("Line")); - CHECK(logValue.isMember("ThreadId")); - CHECK(logValue.isMember("Params")); - CHECK(logValue.isMember("Level")); - CHECK(logValue.isMember("Message")); - CHECK(logValue.isMember("Time")); - - CHECK(logValue["Function"].isString()); - CHECK(logValue["File"].isString()); - CHECK(logValue["Line"].isNumeric()); - CHECK(logValue["Params"].isNull()); - CHECK(logValue["Message"].isString()); - CHECK(logValue["Message"].asString() == "true Test false"); -} - -TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") -{ - { - stream().str(""); - journal().trace() << "Test"; - Json::Value logValue; - Json::Reader reader; - reader.parse(stream().str(), logValue); - - CHECK( - logValue["Level"].asString() == - beast::severities::to_string(beast::severities::kTrace)); - } - - { - stream().str(""); - journal().debug() << "Test"; - Json::Value logValue; - Json::Reader reader; - reader.parse(stream().str(), logValue); - - CHECK( - logValue["Level"].asString() == - beast::severities::to_string(beast::severities::kDebug)); - } - - { - stream().str(""); - journal().info() << "Test"; - Json::Value logValue; - Json::Reader reader; - reader.parse(stream().str(), logValue); - - CHECK( - logValue["Level"].asString() == - beast::severities::to_string(beast::severities::kInfo)); - } - - { - stream().str(""); - journal().warn() << "Test"; - Json::Value logValue; - Json::Reader reader; - reader.parse(stream().str(), logValue); - - CHECK( - logValue["Level"].asString() == - beast::severities::to_string(beast::severities::kWarning)); - } - - { - stream().str(""); - journal().error() << "Test"; - Json::Value logValue; - Json::Reader reader; - reader.parse(stream().str(), logValue); - - CHECK( - logValue["Level"].asString() == - beast::severities::to_string(beast::severities::kError)); - } - - { - stream().str(""); - journal().fatal() << "Test"; - Json::Value logValue; - Json::Reader reader; - reader.parse(stream().str(), logValue); - - CHECK( - logValue["Level"].asString() == - beast::severities::to_string(beast::severities::kFatal)); - } -} - -TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogStream") -{ - journal().stream(beast::severities::kError) << "Test"; - - Json::Value logValue; - Json::Reader reader; - reader.parse(stream().str(), logValue); - - CHECK( - logValue["Level"].asString() == - beast::severities::to_string(beast::severities::kError)); -} - -TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogParams") -{ - journal().debug() << "Test: " << log::param("Field1", 1) << ", " - << log::param( - "Field2", - std::numeric_limits::max()); - - Json::Value logValue; - Json::Reader reader; - reader.parse(stream().str(), logValue); - - CHECK(logValue["Params"].isObject()); - CHECK(logValue["Params"]["Field1"].isNumeric()); - CHECK(logValue["Params"]["Field1"].asInt() == 1); - // UInt64 doesn't fit in Json::Value so it should be converted to a string - // NOTE: We should expect it to be an int64 after we make the json library - // support in64 and uint64 - CHECK(logValue["Params"]["Field2"].isString()); - CHECK(logValue["Params"]["Field2"].asString() == "18446744073709551615"); - CHECK(logValue["Message"].isString()); - CHECK(logValue["Message"].asString() == "Test: 1, 18446744073709551615"); -} - -TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") -{ - journal().debug() << "Test" << log::field("Field1", 1) - << log::field( - "Field2", - std::numeric_limits::max()); - - Json::Value logValue; - Json::Reader reader; - reader.parse(stream().str(), logValue); - - CHECK(logValue["Params"].isObject()); - CHECK(logValue["Params"]["Field1"].isNumeric()); - CHECK(logValue["Params"]["Field1"].asInt() == 1); - // UInt64 doesn't fit in Json::Value so it should be converted to a string - // NOTE: We should expect it to be an int64 after we make the json library - // support in64 and uint64 - CHECK(logValue["Params"]["Field2"].isString()); - CHECK(logValue["Params"]["Field2"].asString() == "18446744073709551615"); - CHECK(logValue["Message"].isString()); - CHECK(logValue["Message"].asString() == "Test"); -} - -TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributes") -{ - beast::Journal j{ - journal(), log::attributes({{"Field1", "Value1"}, {"Field2", 2}})}; - - j.debug() << "Test"; - - Json::Value logValue; - Json::Reader reader; - reader.parse(stream().str(), logValue); - - CHECK(logValue["Field1"].isString()); - CHECK(logValue["Field1"].asString() == "Value1"); - CHECK(logValue["Field2"].isNumeric()); - CHECK(logValue["Field2"].asInt() == 2); -} - -TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributesInheritable") -{ - beast::Journal j{ - journal(), log::attributes({{"Field1", "Value1"}, {"Field2", 2}})}; - beast::Journal j2{ - j, log::attributes({{"Field3", "Value3"}, {"Field2", 0}})}; - - j2.debug() << "Test"; - - Json::Value logValue; - Json::Reader reader; - reader.parse(stream().str(), logValue); - - CHECK(logValue["Field1"].isString()); - CHECK(logValue["Field1"].asString() == "Value1"); - CHECK(logValue["Field3"].isString()); - CHECK(logValue["Field3"].asString() == "Value3"); - // Field2 should be overwritten to 0 - CHECK(logValue["Field2"].isNumeric()); - CHECK(logValue["Field2"].asInt() == 0); -} - -TEST_CASE_FIXTURE( - JsonLogStreamFixture, - "TestJournalAttributesInheritableAfterMoving") -{ - beast::Journal j{ - std::move(journal()), - log::attributes({{"Field1", "Value1"}, {"Field2", 2}})}; - beast::Journal j2{ - std::move(j), log::attributes({{"Field3", "Value3"}, {"Field2", 0}})}; - - j2.debug() << "Test"; - - Json::Value logValue; - Json::Reader reader; - reader.parse(stream().str(), logValue); - - CHECK(logValue["Field1"].isString()); - CHECK(logValue["Field1"].asString() == "Value1"); - CHECK(logValue["Field3"].isString()); - CHECK(logValue["Field3"].asString() == "Value3"); - // Field2 should be overwritten to 0 - CHECK(logValue["Field2"].isNumeric()); - CHECK(logValue["Field2"].asInt() == 0); -} - -TEST_CASE_FIXTURE( - JsonLogStreamFixture, - "TestJournalAttributesInheritableAfterCopyAssignment") -{ - beast::Journal j{ - std::move(journal()), - log::attributes({{"Field1", "Value1"}, {"Field2", 2}})}; - - beast::Journal j2{beast::Journal::getNullSink()}; - - j2 = j; - - j2.debug() << "Test"; - - Json::Value logValue; - Json::Reader reader; - reader.parse(stream().str(), logValue); - - CHECK(logValue["Field1"].isString()); - CHECK(logValue["Field1"].asString() == "Value1"); - CHECK(logValue["Field2"].isNumeric()); - CHECK(logValue["Field2"].asInt() == 2); -} - -TEST_CASE_FIXTURE( - JsonLogStreamFixture, - "TestJournalAttributesInheritableAfterMoveAssignment") -{ - beast::Journal j{ - std::move(journal()), - log::attributes({{"Field1", "Value1"}, {"Field2", 2}})}; - - beast::Journal j2{beast::Journal::getNullSink()}; - - j2 = std::move(j); - - j2.debug() << "Test"; - - Json::Value logValue; - Json::Reader reader; - reader.parse(stream().str(), logValue); - - CHECK(logValue["Field1"].isString()); - CHECK(logValue["Field1"].asString() == "Value1"); - CHECK(logValue["Field2"].isNumeric()); - CHECK(logValue["Field2"].asInt() == 2); -} \ No newline at end of file diff --git a/src/tests/libxrpl/telemetry/main.cpp b/src/tests/libxrpl/telemetry/main.cpp deleted file mode 100644 index 0a3f254ea87..00000000000 --- a/src/tests/libxrpl/telemetry/main.cpp +++ /dev/null @@ -1,2 +0,0 @@ -#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN -#include diff --git a/src/xrpld/app/consensus/RCLConsensus.cpp b/src/xrpld/app/consensus/RCLConsensus.cpp index 47f6cc70f61..b95f8cd5db2 100644 --- a/src/xrpld/app/consensus/RCLConsensus.cpp +++ b/src/xrpld/app/consensus/RCLConsensus.cpp @@ -1109,7 +1109,7 @@ RclConsensusLogger::RclConsensusLogger( bool const validating, beast::Journal j, std::source_location location) - : j_(j, log::attributes({{"Role", "ConsensusLogger"}, {"Label", label}})) + : j_(j, log::attributes(log::attr("Role", "ConsensusLogger"), log::attr("Label", label))) , location_(location) { if (!validating && !j.info()) diff --git a/src/xrpld/app/main/Application.cpp b/src/xrpld/app/main/Application.cpp index 505ded28978..43b389c2703 100644 --- a/src/xrpld/app/main/Application.cpp +++ b/src/xrpld/app/main/Application.cpp @@ -69,7 +69,6 @@ #include #include #include -#include #include #include @@ -835,8 +834,8 @@ class ApplicationImp : public Application, public BasicApp beast::Journal journal( std::string const& name, - std::unique_ptr attributes = - {}) override; + std::optional attributes = + std::nullopt) override; //-------------------------------------------------------------------------- @@ -1220,9 +1219,9 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline) << ", Instance Cookie: " << log::param("InstanceCookie", instanceCookie_); - Logs::setGlobalAttributes(log::attributes( - {{"RippledVersion", BuildInfo::getFullVersionString()}, - {"InstanceCookie", to_string(instanceCookie_)}})); + beast::Journal::addGlobalAttributes(log::attributes( + log::attr("RippledVersion", BuildInfo::getFullVersionString()), + log::attr("InstanceCookie", to_string(instanceCookie_)))); if (numberOfThreads(*config_) < 2) { @@ -2176,7 +2175,7 @@ ApplicationImp::serverOkay(std::string& reason) beast::Journal ApplicationImp::journal( std::string const& name, - std::unique_ptr attributes) + std::optional attributes) { return logs_->journal(name, std::move(attributes)); } diff --git a/src/xrpld/app/main/Application.h b/src/xrpld/app/main/Application.h index f2883a4c394..6b8f5429421 100644 --- a/src/xrpld/app/main/Application.h +++ b/src/xrpld/app/main/Application.h @@ -260,8 +260,8 @@ class Application : public beast::PropertyStream::Source virtual beast::Journal journal( std::string const& name, - std::unique_ptr attributes = - {}) = 0; + std::optional attributes = + std::nullopt) = 0; /* Returns the number of file descriptors the application needs */ virtual int diff --git a/src/xrpld/app/main/Main.cpp b/src/xrpld/app/main/Main.cpp index 1de6d0258e8..1531f8c3f09 100644 --- a/src/xrpld/app/main/Main.cpp +++ b/src/xrpld/app/main/Main.cpp @@ -27,7 +27,6 @@ #include #include #include -#include #include #include @@ -797,10 +796,10 @@ run(int argc, char** argv) if (config->LOG_STYLE == LogStyle::Json) { - auto structuredJournal = std::make_unique(); - beast::Journal::enableStructuredJournal(std::move(structuredJournal)); - Logs::setGlobalAttributes(log::attributes( - {{"Application", "rippled"}, {"NetworkID", config->NETWORK_ID}})); + beast::Journal::enableStructuredJournal(); + beast::Journal::addGlobalAttributes(log::attributes( + log::attr("Application", "rippled"), + log::attr("NetworkID", config->NETWORK_ID))); } auto logs = std::make_unique(thresh); diff --git a/src/xrpld/app/misc/NetworkOPs.cpp b/src/xrpld/app/misc/NetworkOPs.cpp index b9069442f8c..2227090bc57 100644 --- a/src/xrpld/app/misc/NetworkOPs.cpp +++ b/src/xrpld/app/misc/NetworkOPs.cpp @@ -1190,6 +1190,10 @@ NetworkOPsImp::strOperatingMode(OperatingMode const mode, bool const admin) void NetworkOPsImp::submitTransaction(std::shared_ptr const& iTrans) { + beast::Journal journal{ + m_journal, + log::attributes( + log::attr("TransactionID", to_string(iTrans->getTransactionID())))}; if (isNeedNetworkLedger()) { // Nothing we can do if we've never been in sync @@ -1253,6 +1257,9 @@ NetworkOPsImp::submitTransaction(std::shared_ptr const& iTrans) bool NetworkOPsImp::preProcessTransaction(std::shared_ptr& transaction) { + beast::Journal journal{ + m_journal, + log::attributes(log::attr("TransactionID", to_string(transaction->getID())))}; auto const newFlags = app_.getHashRouter().getFlags(transaction->getID()); if ((newFlags & HashRouterFlags::BAD) != HashRouterFlags::UNDEFINED) diff --git a/src/xrpld/app/tx/detail/Transactor.cpp b/src/xrpld/app/tx/detail/Transactor.cpp index f9fb1b5b60b..e1477f7b501 100644 --- a/src/xrpld/app/tx/detail/Transactor.cpp +++ b/src/xrpld/app/tx/detail/Transactor.cpp @@ -209,10 +209,10 @@ Transactor::Transactor(ApplyContext& ctx) : ctx_(ctx) , account_(ctx.tx.getAccountID(sfAccount)) , j_(ctx.journal, - log::attributes({ - {"TransactionID", to_string(ctx_.tx.getTransactionID())}, - {"AccountID", to_string(account_)}, - })) + log::attributes( + log::attr("TransactionID", to_string(ctx_.tx.getTransactionID())), + log::attr("AccountID", to_string(account_)) + )) { } diff --git a/src/xrpld/app/tx/detail/Transactor.h b/src/xrpld/app/tx/detail/Transactor.h index 89feeb8933a..1ac5a690b4f 100644 --- a/src/xrpld/app/tx/detail/Transactor.h +++ b/src/xrpld/app/tx/detail/Transactor.h @@ -53,7 +53,11 @@ struct PreflightContext , rules(rules_) , flags(flags_) , parentBatchId(parentBatchId_) - , j(j_) + , j(j_, + log::attributes( + log::attr("TransactionID", to_string(tx.getTransactionID())), + log::attr("AccountID", to_string(tx.getAccountID(sfAccount))) + )) { XRPL_ASSERT( (flags_ & tapBATCH) == tapBATCH, "Batch apply flag should be set"); @@ -101,7 +105,11 @@ struct PreclaimContext , flags(flags_) , tx(tx_) , parentBatchId(parentBatchId_) - , j(j_) + , j(j_, + log::attributes( + log::attr("TransactionID", to_string(tx.getTransactionID())), + log::attr("AccountID", to_string(tx.getAccountID(sfAccount))) + )) { XRPL_ASSERT( parentBatchId.has_value() == ((flags_ & tapBATCH) == tapBATCH), diff --git a/src/xrpld/core/ClosureCounter.h b/src/xrpld/core/ClosureCounter.h index 92117a91c45..ded056412af 100644 --- a/src/xrpld/core/ClosureCounter.h +++ b/src/xrpld/core/ClosureCounter.h @@ -180,6 +180,13 @@ class ClosureCounter } } + template + Substitute + forceWrap(Closure&& closure) + { + return {*this, std::forward(closure)}; + } + /** Wrap the passed closure with a reference counter. @param closure Closure that accepts Args_t parameters and returns Ret_t. diff --git a/src/xrpld/core/Config.h b/src/xrpld/core/Config.h index c4b26694643..d155976d7e2 100644 --- a/src/xrpld/core/Config.h +++ b/src/xrpld/core/Config.h @@ -25,7 +25,6 @@ #include #include #include // VFALCO Breaks levelization -#include #include // VFALCO FIX: This include should not be here diff --git a/src/xrpld/core/Coro.ipp b/src/xrpld/core/Coro.ipp index 5901e07c684..3f67568e391 100644 --- a/src/xrpld/core/Coro.ipp +++ b/src/xrpld/core/Coro.ipp @@ -98,6 +98,10 @@ JobQueue::Coro::resume() } { std::lock_guard lock(jq_.m_mutex); + + XRPL_ASSERT( + jq_.nSuspend_ > 0, + "ripple::JobQueue::Coro::resume jq_.nSuspend_ should be greater than 0"); --jq_.nSuspend_; } auto saved = detail::getLocalValues().release(); @@ -134,6 +138,10 @@ JobQueue::Coro::expectEarlyExit() // That said, since we're outside the Coro's stack, we need to // decrement the nSuspend that the Coro's call to yield caused. std::lock_guard lock(jq_.m_mutex); + + XRPL_ASSERT( + jq_.nSuspend_ > 0, + "ripple::JobQueue::Coro::expectEarlyExit() jq_.nSuspend_ should be greater than 0"); --jq_.nSuspend_; #ifndef NDEBUG finished_ = true; diff --git a/src/xrpld/core/detail/JobQueue.cpp b/src/xrpld/core/detail/JobQueue.cpp index 1ea1df51abb..ceb449f3b9a 100644 --- a/src/xrpld/core/detail/JobQueue.cpp +++ b/src/xrpld/core/detail/JobQueue.cpp @@ -304,9 +304,9 @@ JobQueue::stop() // but there may still be some threads between the return of // `Job::doJob` and the return of `JobQueue::processTask`. That is why // we must wait on the condition variable to make these assertions. - std::unique_lock lock(m_mutex); + std::unique_lock lock(m_mutex); cv_.wait( - lock, [this] { return m_processCount == 0 && m_jobSet.empty(); }); + lock, [this] { return m_processCount == 0 && nSuspend_ == 0 && m_jobSet.empty(); }); XRPL_ASSERT( m_processCount == 0, "ripple::JobQueue::stop : all processes completed"); diff --git a/src/xrpld/overlay/detail/ConnectAttempt.cpp b/src/xrpld/overlay/detail/ConnectAttempt.cpp index 048933c82cd..4764c063319 100644 --- a/src/xrpld/overlay/detail/ConnectAttempt.cpp +++ b/src/xrpld/overlay/detail/ConnectAttempt.cpp @@ -41,7 +41,7 @@ ConnectAttempt::ConnectAttempt( : Child(overlay) , app_(app) , id_(id) - , journal_(journal, log::attributes({{"NodeID", id}})) + , journal_(journal, log::attributes(log::attr("NodeID", id))) , remote_endpoint_(remote_endpoint) , usage_(usage) , strand_(boost::asio::make_strand(io_context)) diff --git a/src/xrpld/overlay/detail/OverlayImpl.cpp b/src/xrpld/overlay/detail/OverlayImpl.cpp index ee3aa53d7ae..f564fc88331 100644 --- a/src/xrpld/overlay/detail/OverlayImpl.cpp +++ b/src/xrpld/overlay/detail/OverlayImpl.cpp @@ -168,7 +168,7 @@ OverlayImpl::onHandoff( endpoint_type remote_endpoint) { auto const id = next_id_++; - auto journal = app_.journal("Peer", log::attributes({{"NodeID", id}})); + auto journal = app_.journal("Peer", log::attributes(log::attr("NodeID", id))); Handoff handoff; if (processRequest(request, handoff)) diff --git a/src/xrpld/overlay/detail/PeerImp.cpp b/src/xrpld/overlay/detail/PeerImp.cpp index 1b6e611c9e5..3771e042612 100644 --- a/src/xrpld/overlay/detail/PeerImp.cpp +++ b/src/xrpld/overlay/detail/PeerImp.cpp @@ -85,15 +85,15 @@ PeerImp::PeerImp( , journal_( app_.journal("Peer"), log::attributes( - {{"NodeID", id}, - {"RemoteAddress", to_string(slot->remote_endpoint())}, - {"PublicKey", toBase58(TokenType::NodePublic, publicKey)}})) + log::attr("NodeID", id), + log::attr("RemoteAddress", to_string(slot->remote_endpoint())), + log::attr("PublicKey", toBase58(TokenType::NodePublic, publicKey)))) , p_journal_( app_.journal("Protocol"), log::attributes( - {{"NodeID", id}, - {"RemoteAddress", to_string(slot->remote_endpoint())}, - {"PublicKey", toBase58(TokenType::NodePublic, publicKey)}})) + log::attr("NodeID", id), + log::attr("RemoteAddress", to_string(slot->remote_endpoint())), + log::attr("PublicKey", toBase58(TokenType::NodePublic, publicKey)))) , stream_ptr_(std::move(stream_ptr)) , socket_(stream_ptr_->next_layer().socket()) , stream_(*stream_ptr_) @@ -1392,6 +1392,12 @@ PeerImp::handleTransaction( { auto stx = std::make_shared(sit); uint256 txID = stx->getTransactionID(); + beast::Journal protocolJournal{ + p_journal_, + log::attributes( + log::attr("TransactionID", to_string(txID)), + log::attr("RawTransaction", strHex(m->rawtransaction())) + )}; // Charge strongly for attempting to relay a txn with tfInnerBatchTxn // LCOV_EXCL_START diff --git a/src/xrpld/overlay/detail/PeerImp.h b/src/xrpld/overlay/detail/PeerImp.h index feda39031eb..06c1de7971f 100644 --- a/src/xrpld/overlay/detail/PeerImp.h +++ b/src/xrpld/overlay/detail/PeerImp.h @@ -830,15 +830,15 @@ PeerImp::PeerImp( , journal_( app_.journal("Peer"), log::attributes( - {{"NodeID", id}, - {"RemoteAddress", to_string(slot->remote_endpoint())}, - {"PublicKey", toBase58(TokenType::NodePublic, publicKey)}})) + log::attr("NodeID", id), + log::attr("RemoteAddress", to_string(slot->remote_endpoint())), + log::attr("PublicKey", toBase58(TokenType::NodePublic, publicKey)))) , p_journal_( app_.journal("Protocol"), log::attributes( - {{"NodeID", id}, - {"RemoteAddress", to_string(slot->remote_endpoint())}, - {"PublicKey", toBase58(TokenType::NodePublic, publicKey)}})) + log::attr("NodeID", id), + log::attr("RemoteAddress", to_string(slot->remote_endpoint())), + log::attr("PublicKey", toBase58(TokenType::NodePublic, publicKey)))) , stream_ptr_(std::move(stream_ptr)) , socket_(stream_ptr_->next_layer().socket()) , stream_(*stream_ptr_) From dc221de60cf57836d7453f274e1f02fe506843bb Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 28 Aug 2025 21:55:14 +0100 Subject: [PATCH 21/88] Fix issues --- include/xrpl/beast/utility/Journal.h | 106 +++++++++----------- include/xrpl/server/detail/BasePeer.h | 12 +-- src/libxrpl/beast/utility/beast_Journal.cpp | 14 +-- src/test/core/Coroutine_test.cpp | 6 +- src/tests/libxrpl/basics/log.cpp | 97 ++++++++++++------ src/xrpld/app/consensus/RCLConsensus.cpp | 5 +- src/xrpld/app/misc/NetworkOPs.cpp | 3 +- src/xrpld/app/tx/detail/Transactor.cpp | 3 +- src/xrpld/app/tx/detail/Transactor.h | 6 +- src/xrpld/core/Coro.ipp | 6 +- src/xrpld/core/detail/JobQueue.cpp | 5 +- src/xrpld/overlay/detail/OverlayImpl.cpp | 3 +- src/xrpld/overlay/detail/PeerImp.cpp | 15 +-- src/xrpld/overlay/detail/PeerImp.h | 12 ++- 14 files changed, 162 insertions(+), 131 deletions(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 6a232cf57c1..d8f2e369ba6 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -21,13 +21,14 @@ #define BEAST_UTILITY_JOURNAL_H_INCLUDED #include + #include #include -#include +#include #include #include - +#include namespace ripple::log { template @@ -75,7 +76,7 @@ operator<<(std::ostream& os, LogField const& param); template std::ostream& operator<<(std::ostream& os, LogParameter const& param); -} +} // namespace ripple::log namespace beast { @@ -117,7 +118,9 @@ class Journal public: template friend std::ostream& - ripple::log::operator<<(std::ostream& os, ripple::log::LogField const& param); + ripple::log::operator<<( + std::ostream& os, + ripple::log::LogField const& param); template friend std::ostream& @@ -198,7 +201,6 @@ class Journal static thread_local JsonLogContext currentJsonLogContext_; - // Invariant: m_sink always points to a valid Sink Sink* m_sink = nullptr; @@ -206,9 +208,11 @@ class Journal initMessageContext(std::source_location location); static std::string - formatLog(std::string const& message, + formatLog( + std::string const& message, severities::Severity severity, std::optional const& attributes = std::nullopt); + public: //-------------------------------------------------------------------------- @@ -300,10 +304,7 @@ class Journal { public: ScopedStream(ScopedStream const& other) - : ScopedStream( - other.m_attributes, - other.m_sink, - other.m_level) + : ScopedStream(other.m_attributes, other.m_sink, other.m_level) { } @@ -390,10 +391,7 @@ class Journal /** Construct or copy another Stream. */ Stream(Stream const& other) - : Stream( - other.m_attributes, - other.m_sink, - other.m_level) + : Stream(other.m_attributes, other.m_sink, other.m_level) { } @@ -471,8 +469,7 @@ class Journal if (m_attributes.has_value()) m_attributes = JsonLogAttributes::combine( other.m_attributes->contextValues_, - m_attributes->contextValues_ - ); + m_attributes->contextValues_); else m_attributes = other.m_attributes; } @@ -521,8 +518,7 @@ class Journal Stream stream(Severity level) const { - return Stream( - m_attributes, *m_sink, level); + return Stream(m_attributes, *m_sink, level); } /** Returns `true` if any message would be logged at this severity level. @@ -542,10 +538,7 @@ class Journal { if (m_jsonLogsEnabled) initMessageContext(location); - return { - m_attributes, - *m_sink, - severities::kTrace}; + return {m_attributes, *m_sink, severities::kTrace}; } Stream @@ -553,10 +546,7 @@ class Journal { if (m_jsonLogsEnabled) initMessageContext(location); - return { - m_attributes, - *m_sink, - severities::kDebug}; + return {m_attributes, *m_sink, severities::kDebug}; } Stream @@ -564,23 +554,17 @@ class Journal { if (m_jsonLogsEnabled) initMessageContext(location); - return { - m_attributes, - *m_sink, - severities::kInfo}; + return {m_attributes, *m_sink, severities::kInfo}; } Stream warn(std::source_location location = std::source_location::current()) const { - const char* a = "a"; + char const* a = "a"; rapidjson::Value v{a, 1}; if (m_jsonLogsEnabled) initMessageContext(location); - return { - m_attributes, - *m_sink, - severities::kWarning}; + return {m_attributes, *m_sink, severities::kWarning}; } Stream @@ -588,10 +572,7 @@ class Journal { if (m_jsonLogsEnabled) initMessageContext(location); - return { - m_attributes, - *m_sink, - severities::kError}; + return {m_attributes, *m_sink, severities::kError}; } Stream @@ -599,10 +580,7 @@ class Journal { if (m_jsonLogsEnabled) initMessageContext(location); - return { - m_attributes, - *m_sink, - severities::kFatal}; + return {m_attributes, *m_sink, severities::kFatal}; } /** @} */ @@ -614,7 +592,9 @@ class Journal { globalLogAttributes_ = JsonLogAttributes{}; } - globalLogAttributes_ = JsonLogAttributes::combine(globalLogAttributes_->contextValues(), globalLogAttributes.contextValues()); + globalLogAttributes_ = JsonLogAttributes::combine( + globalLogAttributes_->contextValues(), + globalLogAttributes.contextValues()); } }; @@ -725,12 +705,12 @@ using logwstream = basic_logstream; } // namespace beast - namespace ripple::log { namespace detail { template -void setJsonValue( +void +setJsonValue( rapidjson::Value& object, rapidjson::MemoryPoolAllocator<>& allocator, char const* name, @@ -739,7 +719,10 @@ void setJsonValue( { using ValueType = std::decay_t; rapidjson::Value jsonValue; - if constexpr (std::constructible_from&>) + if constexpr (std::constructible_from< + rapidjson::Value, + ValueType, + rapidjson::MemoryPoolAllocator<>&>) { jsonValue = rapidjson::Value{value, allocator}; if (outStream) @@ -777,12 +760,9 @@ void setJsonValue( } object.AddMember( - rapidjson::StringRef(name), - std::move(jsonValue), - allocator - ); -} + rapidjson::StringRef(name), std::move(jsonValue), allocator); } +} // namespace detail template std::ostream& @@ -793,7 +773,9 @@ operator<<(std::ostream& os, LogParameter const& param) detail::setJsonValue( beast::Journal::currentJsonLogContext_.messageParams, beast::Journal::currentJsonLogContext_.allocator, - param.name_, param.value_, &os); + param.name_, + param.value_, + &os); return os; } @@ -806,7 +788,9 @@ operator<<(std::ostream& os, LogField const& param) detail::setJsonValue( beast::Journal::currentJsonLogContext_.messageParams, beast::Journal::currentJsonLogContext_.allocator, - param.name_, param.value_, nullptr); + param.name_, + param.value_, + nullptr); return os; } @@ -824,17 +808,19 @@ field(char const* name, T&& value) return LogField{name, std::forward(value)}; } -template +template [[nodiscard]] beast::Journal::JsonLogAttributes attributes(Pair&&... pairs) { beast::Journal::JsonLogAttributes result; (detail::setJsonValue( - result.contextValues(), - result.allocator(), - pairs.first, - pairs.second, nullptr), ...); + result.contextValues(), + result.allocator(), + pairs.first, + pairs.second, + nullptr), + ...); return result; } @@ -846,6 +832,6 @@ attr(char const* name, T&& value) return std::make_pair(name, std::forward(value)); } -} +} // namespace ripple::log #endif diff --git a/include/xrpl/server/detail/BasePeer.h b/include/xrpl/server/detail/BasePeer.h index 0a578fc65ea..e733b9e4f5c 100644 --- a/include/xrpl/server/detail/BasePeer.h +++ b/include/xrpl/server/detail/BasePeer.h @@ -84,12 +84,12 @@ BasePeer::BasePeer( , handler_(handler) , remote_address_(remote_address) , j_(journal, - log::attributes( - log::attr("PeerID", - [] { - static std::atomic id{0}; - return "##" + std::to_string(++id) + " "; - }()))) + log::attributes(log::attr( + "PeerID", + [] { + static std::atomic id{0}; + return "##" + std::to_string(++id) + " "; + }()))) , work_(boost::asio::make_work_guard(executor)) , strand_(boost::asio::make_strand(executor)) { diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index bcb9c3048e9..122bc11f5e8 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -20,14 +20,14 @@ #include #include -#include #include +#include -#include -#include #include #include +#include #include +#include namespace beast { @@ -151,8 +151,7 @@ Journal::JsonLogAttributes::setModuleName(std::string const& name) contextValues_.AddMember( rapidjson::StringRef("Module"), rapidjson::Value{name.c_str(), allocator_}, - allocator_ - ); + allocator_); } Journal::JsonLogAttributes @@ -166,7 +165,7 @@ Journal::JsonLogAttributes::combine( for (auto& member : b.GetObject()) { - auto val = rapidjson::Value{ member.value, result.allocator_ }; + auto val = rapidjson::Value{member.value, result.allocator_}; if (result.contextValues_.HasMember(member.name)) { result.contextValues_[member.name] = std::move(val); @@ -206,7 +205,8 @@ Journal::formatLog( if (globalLogAttributes_) { - for (auto const& [key, value] : globalLogAttributes_->contextValues().GetObject()) + for (auto const& [key, value] : + globalLogAttributes_->contextValues().GetObject()) { rapidjson::Value jsonValue; jsonValue.CopyFrom(value, currentJsonLogContext_.allocator); diff --git a/src/test/core/Coroutine_test.cpp b/src/test/core/Coroutine_test.cpp index bbdd2e9b001..e4e829919de 100644 --- a/src/test/core/Coroutine_test.cpp +++ b/src/test/core/Coroutine_test.cpp @@ -178,7 +178,6 @@ class Coroutine_test : public beast::unit_test::suite void test_yield_and_stop() { - using namespace std::chrono_literals; using namespace jtx; @@ -230,7 +229,10 @@ class Coroutine_test : public beast::unit_test::suite try { th.join(); - } catch (const std::exception& e) {} + } + catch (std::exception const& e) + { + } pass(); } diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index 5d6e7ca2cca..fb634a3a71b 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -19,8 +19,8 @@ #include -#include #include +#include using namespace ripple; @@ -152,14 +152,16 @@ TEST_CASE("Global attributes") MockLogs logs{logStream, beast::severities::kAll}; beast::Journal::enableStructuredJournal(); - beast::Journal::addGlobalAttributes(log::attributes(log::attr("Field1", "Value1"))); + beast::Journal::addGlobalAttributes( + log::attributes(log::attr("Field1", "Value1"))); logs.journal("Test").debug() << "Test"; rapidjson::Document jsonLog; jsonLog.Parse(logStream.str().c_str()); - CHECK(jsonLog.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + CHECK( + jsonLog.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); CHECK(jsonLog.IsObject()); CHECK(jsonLog.HasMember("Field1")); @@ -175,18 +177,21 @@ TEST_CASE("Global attributes inheritable") MockLogs logs{logStream, beast::severities::kAll}; beast::Journal::enableStructuredJournal(); - beast::Journal::addGlobalAttributes(log::attributes(log::attr("Field1", "Value1"))); + beast::Journal::addGlobalAttributes( + log::attributes(log::attr("Field1", "Value1"))); logs.journal( "Test", - log::attributes(log::attr("Field1", "Value3"), log::attr("Field2", "Value2"))) + log::attributes( + log::attr("Field1", "Value3"), log::attr("Field2", "Value2"))) .debug() << "Test"; rapidjson::Document jsonLog; jsonLog.Parse(logStream.str().c_str()); - CHECK(jsonLog.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + CHECK( + jsonLog.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); CHECK(jsonLog.IsObject()); CHECK(jsonLog.HasMember("Field1")); @@ -265,7 +270,8 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") rapidjson::Document logValue; logValue.Parse(stream().str().c_str()); - CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + CHECK( + logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); CHECK(logValue.IsObject()); CHECK(logValue.HasMember("Function")); @@ -294,7 +300,9 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") rapidjson::Document logValue; logValue.Parse(stream().str().c_str()); - CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + CHECK( + logValue.GetParseError() == + rapidjson::ParseErrorCode::kParseErrorNone); CHECK( logValue["Level"].GetString() == @@ -308,8 +316,9 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") rapidjson::Document logValue; logValue.Parse(stream().str().c_str()); - CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); - + CHECK( + logValue.GetParseError() == + rapidjson::ParseErrorCode::kParseErrorNone); CHECK( logValue["Level"].GetString() == @@ -323,8 +332,9 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") rapidjson::Document logValue; logValue.Parse(stream().str().c_str()); - CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); - + CHECK( + logValue.GetParseError() == + rapidjson::ParseErrorCode::kParseErrorNone); CHECK( logValue["Level"].GetString() == @@ -338,8 +348,9 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") rapidjson::Document logValue; logValue.Parse(stream().str().c_str()); - CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); - + CHECK( + logValue.GetParseError() == + rapidjson::ParseErrorCode::kParseErrorNone); CHECK( logValue["Level"].GetString() == @@ -353,8 +364,9 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") rapidjson::Document logValue; logValue.Parse(stream().str().c_str()); - CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); - + CHECK( + logValue.GetParseError() == + rapidjson::ParseErrorCode::kParseErrorNone); CHECK( logValue["Level"].GetString() == @@ -368,7 +380,9 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") rapidjson::Document logValue; logValue.Parse(stream().str().c_str()); - CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + CHECK( + logValue.GetParseError() == + rapidjson::ParseErrorCode::kParseErrorNone); CHECK( logValue["Level"].GetString() == @@ -383,7 +397,8 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogStream") rapidjson::Document logValue; logValue.Parse(stream().str().c_str()); - CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + CHECK( + logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); CHECK( logValue["Level"].GetString() == @@ -400,7 +415,8 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogParams") rapidjson::Document logValue; logValue.Parse(stream().str().c_str()); - CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + CHECK( + logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); CHECK(logValue["Params"].IsObject()); CHECK(logValue["Params"]["Field1"].IsNumber()); @@ -409,9 +425,13 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogParams") // NOTE: We should expect it to be an int64 after we make the json library // support in64 and uint64 CHECK(logValue["Params"]["Field2"].IsNumber()); - CHECK(logValue["Params"]["Field2"].GetUint64() == std::numeric_limits::max()); + CHECK( + logValue["Params"]["Field2"].GetUint64() == + std::numeric_limits::max()); CHECK(logValue["Message"].IsString()); - CHECK(logValue["Message"].GetString() == std::string{"Test: 1, 18446744073709551615"}); + CHECK( + logValue["Message"].GetString() == + std::string{"Test: 1, 18446744073709551615"}); } TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") @@ -424,8 +444,8 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") rapidjson::Document logValue; logValue.Parse(stream().str().c_str()); - CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); - + CHECK( + logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); CHECK(logValue["Params"].IsObject()); CHECK(logValue["Params"]["Field1"].IsNumber()); @@ -434,7 +454,9 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") // NOTE: We should expect it to be an int64 after we make the json library // support in64 and uint64 CHECK(logValue["Params"]["Field2"].IsNumber()); - CHECK(logValue["Params"]["Field2"].GetUint64() == std::numeric_limits::max()); + CHECK( + logValue["Params"]["Field2"].GetUint64() == + std::numeric_limits::max()); CHECK(logValue["Message"].IsString()); CHECK(logValue["Message"].GetString() == std::string{"Test"}); } @@ -442,14 +464,16 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributes") { beast::Journal j{ - journal(), log::attributes(log::attr("Field1", "Value1"), log::attr("Field2", 2))}; + journal(), + log::attributes(log::attr("Field1", "Value1"), log::attr("Field2", 2))}; j.debug() << "Test"; rapidjson::Document logValue; logValue.Parse(stream().str().c_str()); - CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + CHECK( + logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); CHECK(logValue["Field1"].IsString()); CHECK(logValue["Field1"].GetString() == std::string{"Value1"}); @@ -460,16 +484,19 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributes") TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributesInheritable") { beast::Journal j{ - journal(), log::attributes(log::attr("Field1", "Value1"), log::attr("Field2", 2))}; + journal(), + log::attributes(log::attr("Field1", "Value1"), log::attr("Field2", 2))}; beast::Journal j2{ - j, log::attributes(log::attr("Field3", "Value3"), log::attr("Field2", 0))}; + j, + log::attributes(log::attr("Field3", "Value3"), log::attr("Field2", 0))}; j2.debug() << "Test"; rapidjson::Document logValue; logValue.Parse(stream().str().c_str()); - CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + CHECK( + logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); CHECK(logValue["Field1"].IsString()); CHECK(logValue["Field1"].GetString() == std::string{"Value1"}); @@ -488,14 +515,16 @@ TEST_CASE_FIXTURE( journal(), log::attributes(log::attr("Field1", "Value1"), log::attr("Field2", 2))}; beast::Journal j2{ - j, log::attributes(log::attr("Field3", "Value3"), log::attr("Field2", 0))}; + j, + log::attributes(log::attr("Field3", "Value3"), log::attr("Field2", 0))}; j2.debug() << "Test"; rapidjson::Document logValue; logValue.Parse(stream().str().c_str()); - CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + CHECK( + logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); CHECK(logValue["Field1"].IsString()); CHECK(logValue["Field1"].GetString() == std::string{"Value1"}); @@ -523,7 +552,8 @@ TEST_CASE_FIXTURE( rapidjson::Document logValue; logValue.Parse(stream().str().c_str()); - CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + CHECK( + logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); CHECK(logValue["Field1"].IsString()); CHECK(logValue["Field1"].GetString() == std::string{"Value1"}); @@ -548,7 +578,8 @@ TEST_CASE_FIXTURE( rapidjson::Document logValue; logValue.Parse(stream().str().c_str()); - CHECK(logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + CHECK( + logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); CHECK(logValue["Field1"].IsString()); CHECK(logValue["Field1"].GetString() == std::string{"Value1"}); diff --git a/src/xrpld/app/consensus/RCLConsensus.cpp b/src/xrpld/app/consensus/RCLConsensus.cpp index b95f8cd5db2..431b9e6bc24 100644 --- a/src/xrpld/app/consensus/RCLConsensus.cpp +++ b/src/xrpld/app/consensus/RCLConsensus.cpp @@ -1109,7 +1109,10 @@ RclConsensusLogger::RclConsensusLogger( bool const validating, beast::Journal j, std::source_location location) - : j_(j, log::attributes(log::attr("Role", "ConsensusLogger"), log::attr("Label", label))) + : j_(j, + log::attributes( + log::attr("Role", "ConsensusLogger"), + log::attr("Label", label))) , location_(location) { if (!validating && !j.info()) diff --git a/src/xrpld/app/misc/NetworkOPs.cpp b/src/xrpld/app/misc/NetworkOPs.cpp index 2227090bc57..b99ae7d6a68 100644 --- a/src/xrpld/app/misc/NetworkOPs.cpp +++ b/src/xrpld/app/misc/NetworkOPs.cpp @@ -1259,7 +1259,8 @@ NetworkOPsImp::preProcessTransaction(std::shared_ptr& transaction) { beast::Journal journal{ m_journal, - log::attributes(log::attr("TransactionID", to_string(transaction->getID())))}; + log::attributes( + log::attr("TransactionID", to_string(transaction->getID())))}; auto const newFlags = app_.getHashRouter().getFlags(transaction->getID()); if ((newFlags & HashRouterFlags::BAD) != HashRouterFlags::UNDEFINED) diff --git a/src/xrpld/app/tx/detail/Transactor.cpp b/src/xrpld/app/tx/detail/Transactor.cpp index e1477f7b501..a48643b2baa 100644 --- a/src/xrpld/app/tx/detail/Transactor.cpp +++ b/src/xrpld/app/tx/detail/Transactor.cpp @@ -211,8 +211,7 @@ Transactor::Transactor(ApplyContext& ctx) , j_(ctx.journal, log::attributes( log::attr("TransactionID", to_string(ctx_.tx.getTransactionID())), - log::attr("AccountID", to_string(account_)) - )) + log::attr("AccountID", to_string(account_)))) { } diff --git a/src/xrpld/app/tx/detail/Transactor.h b/src/xrpld/app/tx/detail/Transactor.h index 1ac5a690b4f..f95d396b908 100644 --- a/src/xrpld/app/tx/detail/Transactor.h +++ b/src/xrpld/app/tx/detail/Transactor.h @@ -56,8 +56,7 @@ struct PreflightContext , j(j_, log::attributes( log::attr("TransactionID", to_string(tx.getTransactionID())), - log::attr("AccountID", to_string(tx.getAccountID(sfAccount))) - )) + log::attr("AccountID", to_string(tx.getAccountID(sfAccount))))) { XRPL_ASSERT( (flags_ & tapBATCH) == tapBATCH, "Batch apply flag should be set"); @@ -108,8 +107,7 @@ struct PreclaimContext , j(j_, log::attributes( log::attr("TransactionID", to_string(tx.getTransactionID())), - log::attr("AccountID", to_string(tx.getAccountID(sfAccount))) - )) + log::attr("AccountID", to_string(tx.getAccountID(sfAccount))))) { XRPL_ASSERT( parentBatchId.has_value() == ((flags_ & tapBATCH) == tapBATCH), diff --git a/src/xrpld/core/Coro.ipp b/src/xrpld/core/Coro.ipp index 3f67568e391..9abf4bbf811 100644 --- a/src/xrpld/core/Coro.ipp +++ b/src/xrpld/core/Coro.ipp @@ -101,7 +101,8 @@ JobQueue::Coro::resume() XRPL_ASSERT( jq_.nSuspend_ > 0, - "ripple::JobQueue::Coro::resume jq_.nSuspend_ should be greater than 0"); + "ripple::JobQueue::Coro::resume jq_.nSuspend_ should be greater " + "than 0"); --jq_.nSuspend_; } auto saved = detail::getLocalValues().release(); @@ -141,7 +142,8 @@ JobQueue::Coro::expectEarlyExit() XRPL_ASSERT( jq_.nSuspend_ > 0, - "ripple::JobQueue::Coro::expectEarlyExit() jq_.nSuspend_ should be greater than 0"); + "ripple::JobQueue::Coro::expectEarlyExit() jq_.nSuspend_ should be " + "greater than 0"); --jq_.nSuspend_; #ifndef NDEBUG finished_ = true; diff --git a/src/xrpld/core/detail/JobQueue.cpp b/src/xrpld/core/detail/JobQueue.cpp index ceb449f3b9a..986bb68545f 100644 --- a/src/xrpld/core/detail/JobQueue.cpp +++ b/src/xrpld/core/detail/JobQueue.cpp @@ -305,8 +305,9 @@ JobQueue::stop() // `Job::doJob` and the return of `JobQueue::processTask`. That is why // we must wait on the condition variable to make these assertions. std::unique_lock lock(m_mutex); - cv_.wait( - lock, [this] { return m_processCount == 0 && nSuspend_ == 0 && m_jobSet.empty(); }); + cv_.wait(lock, [this] { + return m_processCount == 0 && nSuspend_ == 0 && m_jobSet.empty(); + }); XRPL_ASSERT( m_processCount == 0, "ripple::JobQueue::stop : all processes completed"); diff --git a/src/xrpld/overlay/detail/OverlayImpl.cpp b/src/xrpld/overlay/detail/OverlayImpl.cpp index f564fc88331..ad7e5f3897e 100644 --- a/src/xrpld/overlay/detail/OverlayImpl.cpp +++ b/src/xrpld/overlay/detail/OverlayImpl.cpp @@ -168,7 +168,8 @@ OverlayImpl::onHandoff( endpoint_type remote_endpoint) { auto const id = next_id_++; - auto journal = app_.journal("Peer", log::attributes(log::attr("NodeID", id))); + auto journal = + app_.journal("Peer", log::attributes(log::attr("NodeID", id))); Handoff handoff; if (processRequest(request, handoff)) diff --git a/src/xrpld/overlay/detail/PeerImp.cpp b/src/xrpld/overlay/detail/PeerImp.cpp index 3771e042612..587e29b6052 100644 --- a/src/xrpld/overlay/detail/PeerImp.cpp +++ b/src/xrpld/overlay/detail/PeerImp.cpp @@ -86,14 +86,18 @@ PeerImp::PeerImp( app_.journal("Peer"), log::attributes( log::attr("NodeID", id), - log::attr("RemoteAddress", to_string(slot->remote_endpoint())), - log::attr("PublicKey", toBase58(TokenType::NodePublic, publicKey)))) + log::attr("RemoteAddress", to_string(slot->remote_endpoint())), + log::attr( + "PublicKey", + toBase58(TokenType::NodePublic, publicKey)))) , p_journal_( app_.journal("Protocol"), log::attributes( log::attr("NodeID", id), - log::attr("RemoteAddress", to_string(slot->remote_endpoint())), - log::attr("PublicKey", toBase58(TokenType::NodePublic, publicKey)))) + log::attr("RemoteAddress", to_string(slot->remote_endpoint())), + log::attr( + "PublicKey", + toBase58(TokenType::NodePublic, publicKey)))) , stream_ptr_(std::move(stream_ptr)) , socket_(stream_ptr_->next_layer().socket()) , stream_(*stream_ptr_) @@ -1396,8 +1400,7 @@ PeerImp::handleTransaction( p_journal_, log::attributes( log::attr("TransactionID", to_string(txID)), - log::attr("RawTransaction", strHex(m->rawtransaction())) - )}; + log::attr("RawTransaction", strHex(m->rawtransaction())))}; // Charge strongly for attempting to relay a txn with tfInnerBatchTxn // LCOV_EXCL_START diff --git a/src/xrpld/overlay/detail/PeerImp.h b/src/xrpld/overlay/detail/PeerImp.h index 06c1de7971f..7e8f96368b5 100644 --- a/src/xrpld/overlay/detail/PeerImp.h +++ b/src/xrpld/overlay/detail/PeerImp.h @@ -831,14 +831,18 @@ PeerImp::PeerImp( app_.journal("Peer"), log::attributes( log::attr("NodeID", id), - log::attr("RemoteAddress", to_string(slot->remote_endpoint())), - log::attr("PublicKey", toBase58(TokenType::NodePublic, publicKey)))) + log::attr("RemoteAddress", to_string(slot->remote_endpoint())), + log::attr( + "PublicKey", + toBase58(TokenType::NodePublic, publicKey)))) , p_journal_( app_.journal("Protocol"), log::attributes( log::attr("NodeID", id), - log::attr("RemoteAddress", to_string(slot->remote_endpoint())), - log::attr("PublicKey", toBase58(TokenType::NodePublic, publicKey)))) + log::attr("RemoteAddress", to_string(slot->remote_endpoint())), + log::attr( + "PublicKey", + toBase58(TokenType::NodePublic, publicKey)))) , stream_ptr_(std::move(stream_ptr)) , socket_(stream_ptr_->next_layer().socket()) , stream_(*stream_ptr_) From 76bb517eb84a106d8a8715a94edc87ebce381b61 Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 28 Aug 2025 22:41:41 +0100 Subject: [PATCH 22/88] Fix issues --- src/test/core/Coroutine_test.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/core/Coroutine_test.cpp b/src/test/core/Coroutine_test.cpp index e4e829919de..a3eb3c94404 100644 --- a/src/test/core/Coroutine_test.cpp +++ b/src/test/core/Coroutine_test.cpp @@ -239,10 +239,10 @@ class Coroutine_test : public beast::unit_test::suite void run() override { - // correct_order(); - // incorrect_order(); - // thread_specific_storage(); - test_yield_and_stop(); + correct_order(); + incorrect_order(); + thread_specific_storage(); + // test_yield_and_stop(); } }; From a128571ab5752165909b6a35c80fba1aef0921a1 Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 28 Aug 2025 23:04:56 +0100 Subject: [PATCH 23/88] Fix issues --- include/xrpl/beast/utility/Journal.h | 7 +++ src/libxrpl/beast/utility/beast_Journal.cpp | 62 +++++++++++---------- 2 files changed, 41 insertions(+), 28 deletions(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index d8f2e369ba6..d2d9ae0d577 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -584,6 +584,13 @@ class Journal } /** @} */ + static void + resetGlobalAttributes() + { + std::lock_guard lock(globalLogAttributesMutex_); + globalLogAttributes_ = std::nullopt; + } + static void addGlobalAttributes(JsonLogAttributes globalLogAttributes) { diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index 122bc11f5e8..a710e7754c1 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -203,34 +203,6 @@ Journal::formatLog( rapidjson::Value logContext; logContext.SetObject(); - if (globalLogAttributes_) - { - for (auto const& [key, value] : - globalLogAttributes_->contextValues().GetObject()) - { - rapidjson::Value jsonValue; - jsonValue.CopyFrom(value, currentJsonLogContext_.allocator); - - logContext.AddMember( - rapidjson::Value{key, currentJsonLogContext_.allocator}, - std::move(jsonValue), - currentJsonLogContext_.allocator); - } - } - - if (attributes.has_value()) - { - for (auto const& [key, value] : attributes->contextValues().GetObject()) - { - rapidjson::Value jsonValue; - jsonValue.CopyFrom(value, currentJsonLogContext_.allocator); - - logContext.AddMember( - rapidjson::Value{key, currentJsonLogContext_.allocator}, - std::move(jsonValue), - currentJsonLogContext_.allocator); - } - } logContext.AddMember( rapidjson::StringRef("Function"), rapidjson::StringRef(currentJsonLogContext_.location.function_name()), @@ -274,6 +246,39 @@ Journal::formatLog( .count(), currentJsonLogContext_.allocator); + if (attributes.has_value()) + { + for (auto const& [key, value] : attributes->contextValues().GetObject()) + { + if (logContext.HasMember(key)) + continue; + rapidjson::Value jsonValue; + jsonValue.CopyFrom(value, currentJsonLogContext_.allocator); + + logContext.AddMember( + rapidjson::Value{key, currentJsonLogContext_.allocator}, + std::move(jsonValue), + currentJsonLogContext_.allocator); + } + } + + if (globalLogAttributes_) + { + for (auto const& [key, value] : + globalLogAttributes_->contextValues().GetObject()) + { + if (logContext.HasMember(key)) + continue; + rapidjson::Value jsonValue; + jsonValue.CopyFrom(value, currentJsonLogContext_.allocator); + + logContext.AddMember( + rapidjson::Value{key, currentJsonLogContext_.allocator}, + std::move(jsonValue), + currentJsonLogContext_.allocator); + } + } + rapidjson::StringBuffer buffer; rapidjson::Writer writer(buffer); @@ -292,6 +297,7 @@ void Journal::disableStructuredJournal() { m_jsonLogsEnabled = false; + resetGlobalAttributes(); } bool From e19d770b86f9db3adae60a2f60f544eccaf0ffdd Mon Sep 17 00:00:00 2001 From: JCW Date: Fri, 29 Aug 2025 22:19:35 +0100 Subject: [PATCH 24/88] performance optimisation Signed-off-by: JCW --- include/xrpl/basics/Log.h | 7 +- include/xrpl/beast/utility/Journal.h | 134 ++++++------- src/libxrpl/basics/Log.cpp | 32 +-- src/libxrpl/beast/utility/beast_Journal.cpp | 207 ++++++++++---------- 4 files changed, 188 insertions(+), 192 deletions(-) diff --git a/include/xrpl/basics/Log.h b/include/xrpl/basics/Log.h index 6638da7bee1..5ba31f89f0c 100644 --- a/include/xrpl/basics/Log.h +++ b/include/xrpl/basics/Log.h @@ -156,9 +156,10 @@ class Logs private: std::unique_ptr m_stream; boost::filesystem::path m_path; + std::mutex mutable fileMutex_; }; - std::mutex mutable mutex_; + std::mutex mutable sinkSetMutex_; std::map< std::string, std::unique_ptr, @@ -205,7 +206,7 @@ class Logs write( beast::severities::Severity level, std::string const& partition, - std::string const& text, + std::string text, bool console); std::string @@ -243,7 +244,7 @@ class Logs static void format( std::string& output, - std::string const& message, + std::string message, beast::severities::Severity severity, std::string const& partition); diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index d2d9ae0d577..f54b57815ef 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -144,8 +144,8 @@ class Journal void setModuleName(std::string const& name); - [[nodiscard]] static JsonLogAttributes - combine(AttributeFields const& a, AttributeFields const& b); + void + combine(AttributeFields const& from); AttributeFields& contextValues() @@ -172,22 +172,37 @@ class Journal friend class Journal; }; - struct JsonLogContext + class JsonLogContext { - std::source_location location = {}; - rapidjson::Value messageParams; - rapidjson::MemoryPoolAllocator<> allocator; + rapidjson::Value attributes_; + rapidjson::MemoryPoolAllocator<> allocator_; + public: JsonLogContext() = default; - void - reset(std::source_location location_) noexcept + rapidjson::MemoryPoolAllocator<>& + allocator() + { + return allocator_; + } + + rapidjson::Value& + messageParams() { - location = location_; - messageParams = rapidjson::Value{}; - messageParams.SetObject(); - allocator.Clear(); + return attributes_["Params"]; } + + rapidjson::Value& + attributes() + { + return attributes_; + } + + void + reset( + std::source_location location, + severities::Severity severity, + std::optional const& attributes) noexcept; }; private: @@ -204,14 +219,13 @@ class Journal // Invariant: m_sink always points to a valid Sink Sink* m_sink = nullptr; - static void - initMessageContext(std::source_location location); + void + initMessageContext( + std::source_location location, + severities::Severity severity) const; static std::string - formatLog( - std::string const& message, - severities::Severity severity, - std::optional const& attributes = std::nullopt); + formatLog(std::string const& message); public: //-------------------------------------------------------------------------- @@ -304,25 +318,16 @@ class Journal { public: ScopedStream(ScopedStream const& other) - : ScopedStream(other.m_attributes, other.m_sink, other.m_level) + : ScopedStream(other.m_sink, other.m_level) { } - ScopedStream( - std::optional attributes, - Sink& sink, - Severity level); + ScopedStream(Sink& sink, Severity level); template - ScopedStream( - std::optional attributes, - Stream const& stream, - T const& t); + ScopedStream(Stream const& stream, T const& t); - ScopedStream( - std::optional attributes, - Stream const& stream, - std::ostream& manip(std::ostream&)); + ScopedStream(Stream const& stream, std::ostream& manip(std::ostream&)); ScopedStream& operator=(ScopedStream const&) = delete; @@ -343,7 +348,6 @@ class Journal operator<<(T const& t) const; private: - std::optional m_attributes; Sink& m_sink; Severity const m_level; std::ostringstream mutable m_ostream; @@ -378,11 +382,7 @@ class Journal Constructor is inlined so checking active() very inexpensive. */ - Stream( - std::optional attributes, - Sink& sink, - Severity level) - : m_attributes(std::move(attributes)), m_sink(sink), m_level(level) + Stream(Sink& sink, Severity level) : m_sink(sink), m_level(level) { XRPL_ASSERT( m_level < severities::kDisabled, @@ -390,8 +390,7 @@ class Journal } /** Construct or copy another Stream. */ - Stream(Stream const& other) - : Stream(other.m_attributes, other.m_sink, other.m_level) + Stream(Stream const& other) : Stream(other.m_sink, other.m_level) { } @@ -438,7 +437,6 @@ class Journal /** @} */ private: - std::optional m_attributes; Sink& m_sink; Severity m_level; }; @@ -467,9 +465,8 @@ class Journal if (other.m_attributes.has_value()) { if (m_attributes.has_value()) - m_attributes = JsonLogAttributes::combine( - other.m_attributes->contextValues_, - m_attributes->contextValues_); + m_attributes->combine( + other.m_attributes->contextValues_); else m_attributes = other.m_attributes; } @@ -516,9 +513,11 @@ class Journal /** Returns a stream for this sink, with the specified severity level. */ Stream - stream(Severity level) const + stream(Severity level, std::source_location location = std::source_location::current()) const { - return Stream(m_attributes, *m_sink, level); + if (m_jsonLogsEnabled) + initMessageContext(location, level); + return Stream(*m_sink, level); } /** Returns `true` if any message would be logged at this severity level. @@ -537,24 +536,24 @@ class Journal trace(std::source_location location = std::source_location::current()) const { if (m_jsonLogsEnabled) - initMessageContext(location); - return {m_attributes, *m_sink, severities::kTrace}; + initMessageContext(location, severities::kTrace); + return {*m_sink, severities::kTrace}; } Stream debug(std::source_location location = std::source_location::current()) const { if (m_jsonLogsEnabled) - initMessageContext(location); - return {m_attributes, *m_sink, severities::kDebug}; + initMessageContext(location, severities::kDebug); + return {*m_sink, severities::kDebug}; } Stream info(std::source_location location = std::source_location::current()) const { if (m_jsonLogsEnabled) - initMessageContext(location); - return {m_attributes, *m_sink, severities::kInfo}; + initMessageContext(location, severities::kInfo); + return {*m_sink, severities::kInfo}; } Stream @@ -563,24 +562,24 @@ class Journal char const* a = "a"; rapidjson::Value v{a, 1}; if (m_jsonLogsEnabled) - initMessageContext(location); - return {m_attributes, *m_sink, severities::kWarning}; + initMessageContext(location, severities::kWarning); + return {*m_sink, severities::kWarning}; } Stream error(std::source_location location = std::source_location::current()) const { if (m_jsonLogsEnabled) - initMessageContext(location); - return {m_attributes, *m_sink, severities::kError}; + initMessageContext(location, severities::kError); + return {*m_sink, severities::kError}; } Stream fatal(std::source_location location = std::source_location::current()) const { if (m_jsonLogsEnabled) - initMessageContext(location); - return {m_attributes, *m_sink, severities::kFatal}; + initMessageContext(location, severities::kFatal); + return {*m_sink, severities::kFatal}; } /** @} */ @@ -599,8 +598,7 @@ class Journal { globalLogAttributes_ = JsonLogAttributes{}; } - globalLogAttributes_ = JsonLogAttributes::combine( - globalLogAttributes_->contextValues(), + globalLogAttributes_->combine( globalLogAttributes.contextValues()); } }; @@ -617,11 +615,8 @@ static_assert(std::is_nothrow_destructible::value == true, ""); //------------------------------------------------------------------------------ template -Journal::ScopedStream::ScopedStream( - std::optional attributes, - Stream const& stream, - T const& t) - : ScopedStream(std::move(attributes), stream.sink(), stream.level()) +Journal::ScopedStream::ScopedStream(Stream const& stream, T const& t) + : ScopedStream(stream.sink(), stream.level()) { m_ostream << t; } @@ -640,7 +635,7 @@ template Journal::ScopedStream Journal::Stream::operator<<(T const& t) const { - return {m_attributes, *this, t}; + return {*this, t}; } namespace detail { @@ -766,6 +761,7 @@ setJsonValue( } } + object.RemoveMember(name); object.AddMember( rapidjson::StringRef(name), std::move(jsonValue), allocator); } @@ -778,8 +774,8 @@ operator<<(std::ostream& os, LogParameter const& param) if (!beast::Journal::m_jsonLogsEnabled) return os; detail::setJsonValue( - beast::Journal::currentJsonLogContext_.messageParams, - beast::Journal::currentJsonLogContext_.allocator, + beast::Journal::currentJsonLogContext_.messageParams(), + beast::Journal::currentJsonLogContext_.allocator(), param.name_, param.value_, &os); @@ -793,8 +789,8 @@ operator<<(std::ostream& os, LogField const& param) if (!beast::Journal::m_jsonLogsEnabled) return os; detail::setJsonValue( - beast::Journal::currentJsonLogContext_.messageParams, - beast::Journal::currentJsonLogContext_.allocator, + beast::Journal::currentJsonLogContext_.messageParams(), + beast::Journal::currentJsonLogContext_.allocator(), param.name_, param.value_, nullptr); diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index 0ddd56f17d2..a87d9bb315c 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -88,9 +88,13 @@ Logs::File::open(boost::filesystem::path const& path) if (stream->good()) { + std::lock_guard lock(fileMutex_); m_path = path; m_stream = std::move(stream); + size_t const bufsize = 256 * 1024; + static char buf[bufsize]; + m_stream->rdbuf()->pubsetbuf(buf, bufsize); wasOpened = true; } @@ -109,12 +113,14 @@ Logs::File::closeAndReopen() void Logs::File::close() { + std::lock_guard lock(fileMutex_); m_stream = nullptr; } void Logs::File::write(char const* text) { + std::lock_guard lock(fileMutex_); if (m_stream != nullptr) (*m_stream) << text; } @@ -122,10 +128,10 @@ Logs::File::write(char const* text) void Logs::File::writeln(char const* text) { + std::lock_guard lock(fileMutex_); if (m_stream != nullptr) { - (*m_stream) << text; - (*m_stream) << std::endl; + (*m_stream) << text << '\n'; } } @@ -145,7 +151,7 @@ Logs::open(boost::filesystem::path const& pathToLogFile) beast::Journal::Sink& Logs::get(std::string const& name) { - std::lock_guard lock(mutex_); + std::lock_guard lock(sinkSetMutex_); auto const result = sinks_.emplace(name, makeSink(name, thresh_)); return *result.first->second; } @@ -173,7 +179,7 @@ Logs::threshold() const void Logs::threshold(beast::severities::Severity thresh) { - std::lock_guard lock(mutex_); + std::lock_guard lock(sinkSetMutex_); thresh_ = thresh; for (auto& sink : sinks_) sink.second->threshold(thresh); @@ -183,7 +189,7 @@ std::vector> Logs::partition_severities() const { std::vector> list; - std::lock_guard lock(mutex_); + std::lock_guard lock(sinkSetMutex_); list.reserve(sinks_.size()); for (auto const& [name, sink] : sinks_) list.emplace_back(name, toString(fromSeverity(sink->threshold()))); @@ -194,12 +200,11 @@ void Logs::write( beast::severities::Severity level, std::string const& partition, - std::string const& text, + std::string text, bool console) { std::string s; - format(s, text, level, partition); - std::lock_guard lock(mutex_); + format(s, std::move(text), level, partition); file_.writeln(s); if (!silent_) std::cerr << s << '\n'; @@ -211,7 +216,6 @@ Logs::write( std::string Logs::rotate() { - std::lock_guard lock(mutex_); bool const wasOpened = file_.closeAndReopen(); if (wasOpened) return "The log file was closed and reopened."; @@ -328,15 +332,15 @@ Logs::fromString(std::string const& s) void Logs::format( std::string& output, - std::string const& message, + std::string message, beast::severities::Severity severity, std::string const& partition) { - output.reserve(message.size() + partition.size() + 100); - + output = std::move(message); if (!beast::Journal::isStructuredJournalEnabled()) { - output = to_string(std::chrono::system_clock::now()); + output.reserve(message.size() + partition.size() + 100); + output += to_string(std::chrono::system_clock::now()); output += " "; if (!partition.empty()) @@ -369,8 +373,6 @@ Logs::format( } } - output += message; - // Limit the maximum length of the output if (output.size() > maximumMessageCharacters) { diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index a710e7754c1..04e286b8db2 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -154,135 +154,136 @@ Journal::JsonLogAttributes::setModuleName(std::string const& name) allocator_); } -Journal::JsonLogAttributes +void Journal::JsonLogAttributes::combine( - AttributeFields const& a, - AttributeFields const& b) + AttributeFields const& from) { - JsonLogAttributes result; + for (auto& member : from.GetObject()) + { + contextValues_.RemoveMember(member.name); - result.contextValues_.CopyFrom(a, result.allocator_); + contextValues_.AddMember( + rapidjson::Value{member.name, allocator_}, + rapidjson::Value{member.value, allocator_}, + allocator_); + } +} - for (auto& member : b.GetObject()) +void +Journal::JsonLogContext::reset( + std::source_location location, + severities::Severity severity, + std::optional const& attributes) noexcept +{ + struct ThreadIdStringInitializer { - auto val = rapidjson::Value{member.value, result.allocator_}; - if (result.contextValues_.HasMember(member.name)) + std::string value; + ThreadIdStringInitializer() { - result.contextValues_[member.name] = std::move(val); + std::stringstream threadIdStream; + threadIdStream << std::this_thread::get_id(); + value = threadIdStream.str(); } - else + }; + thread_local ThreadIdStringInitializer threadId; + + attributes_.SetObject(); + if (globalLogAttributes_.has_value()) + { + attributes_.CopyFrom(globalLogAttributes_->contextValues(), allocator_); + if (attributes.has_value()) { - result.contextValues_.AddMember( - rapidjson::Value{member.name, result.allocator_}, - std::move(val), - result.allocator_); + for (auto const& [key, value] : + attributes->contextValues().GetObject()) + { + attributes_.RemoveMember(key); + + rapidjson::Value jsonValue; + jsonValue.CopyFrom(value, allocator_); + + attributes_.AddMember( + rapidjson::Value{key, allocator_}, + rapidjson::Value{value, allocator_}, + allocator_); + } } } - - return result; -} - -void -Journal::initMessageContext(std::source_location location) -{ - currentJsonLogContext_.reset(location); -} - -std::string -Journal::formatLog( - std::string const& message, - severities::Severity severity, - std::optional const& attributes) -{ - if (!m_jsonLogsEnabled) + else if (attributes.has_value()) { - return message; + attributes_.CopyFrom(attributes->contextValues(), allocator_); } - rapidjson::Document doc{¤tJsonLogContext_.allocator}; - rapidjson::Value logContext; - logContext.SetObject(); - - logContext.AddMember( + attributes_.RemoveMember("Function"); + attributes_.AddMember( rapidjson::StringRef("Function"), - rapidjson::StringRef(currentJsonLogContext_.location.function_name()), - currentJsonLogContext_.allocator); + rapidjson::Value{location.function_name(), allocator_}, + allocator_); - logContext.AddMember( + attributes_.RemoveMember("File"); + attributes_.AddMember( rapidjson::StringRef("File"), - rapidjson::StringRef(currentJsonLogContext_.location.file_name()), - currentJsonLogContext_.allocator); + rapidjson::Value{location.file_name(), allocator_}, + allocator_); - logContext.AddMember( + attributes_.RemoveMember("Line"); + attributes_.AddMember( rapidjson::StringRef("Line"), - currentJsonLogContext_.location.line(), - currentJsonLogContext_.allocator); - std::stringstream threadIdStream; - threadIdStream << std::this_thread::get_id(); - auto threadIdStr = threadIdStream.str(); - logContext.AddMember( + location.line(), + allocator_); + attributes_.RemoveMember("ThreadId"); + attributes_.AddMember( rapidjson::StringRef("ThreadId"), - rapidjson::StringRef(threadIdStr.c_str()), - currentJsonLogContext_.allocator); - logContext.AddMember( + rapidjson::Value{threadId.value.c_str(), allocator_}, + allocator_); + attributes_.RemoveMember("Params"); + attributes_.AddMember( rapidjson::StringRef("Params"), - std::move(currentJsonLogContext_.messageParams), - currentJsonLogContext_.allocator); - currentJsonLogContext_.messageParams = rapidjson::Value{}; - currentJsonLogContext_.messageParams.SetObject(); + rapidjson::Value{rapidjson::kObjectType}, + allocator_); auto severityStr = to_string(severity); - logContext.AddMember( + attributes_.RemoveMember("Level"); + attributes_.AddMember( rapidjson::StringRef("Level"), - rapidjson::StringRef(severityStr.c_str()), - currentJsonLogContext_.allocator); - logContext.AddMember( - rapidjson::StringRef("Message"), - rapidjson::StringRef(message.c_str()), - currentJsonLogContext_.allocator); - logContext.AddMember( + rapidjson::Value{severityStr.c_str(), allocator_}, + allocator_); + attributes_.RemoveMember("Time"); + attributes_.AddMember( rapidjson::StringRef("Time"), std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(), - currentJsonLogContext_.allocator); + allocator_); +} - if (attributes.has_value()) - { - for (auto const& [key, value] : attributes->contextValues().GetObject()) - { - if (logContext.HasMember(key)) - continue; - rapidjson::Value jsonValue; - jsonValue.CopyFrom(value, currentJsonLogContext_.allocator); - - logContext.AddMember( - rapidjson::Value{key, currentJsonLogContext_.allocator}, - std::move(jsonValue), - currentJsonLogContext_.allocator); - } - } +void +Journal::initMessageContext( + std::source_location location, + severities::Severity severity) const +{ + currentJsonLogContext_.reset(location, severity, m_attributes); +} - if (globalLogAttributes_) +std::string +Journal::formatLog(std::string const& message) +{ + if (!m_jsonLogsEnabled) { - for (auto const& [key, value] : - globalLogAttributes_->contextValues().GetObject()) - { - if (logContext.HasMember(key)) - continue; - rapidjson::Value jsonValue; - jsonValue.CopyFrom(value, currentJsonLogContext_.allocator); - - logContext.AddMember( - rapidjson::Value{key, currentJsonLogContext_.allocator}, - std::move(jsonValue), - currentJsonLogContext_.allocator); - } + return message; } + auto& attributes = currentJsonLogContext_.attributes(); + + attributes.RemoveMember("Message"); + attributes.AddMember( + rapidjson::StringRef("Message"), + rapidjson::Value{rapidjson::StringRef(message.c_str()), currentJsonLogContext_.allocator()}, + currentJsonLogContext_.allocator() + ); + rapidjson::StringBuffer buffer; rapidjson::Writer writer(buffer); - logContext.Accept(writer); + attributes.Accept(writer); return {buffer.GetString()}; } @@ -345,21 +346,17 @@ Journal::Sink::threshold(Severity thresh) //------------------------------------------------------------------------------ -Journal::ScopedStream::ScopedStream( - std::optional attributes, - Sink& sink, - Severity level) - : m_attributes(std::move(attributes)), m_sink(sink), m_level(level) +Journal::ScopedStream::ScopedStream(Sink& sink, Severity level) + : m_sink(sink), m_level(level) { // Modifiers applied from all ctors m_ostream << std::boolalpha << std::showbase; } Journal::ScopedStream::ScopedStream( - std::optional attributes, Stream const& stream, std::ostream& manip(std::ostream&)) - : ScopedStream(std::move(attributes), stream.sink(), stream.level()) + : ScopedStream(stream.sink(), stream.level()) { m_ostream << manip; } @@ -370,9 +367,9 @@ Journal::ScopedStream::~ScopedStream() if (!s.empty()) { if (s == "\n") - m_sink.write(m_level, formatLog("", m_level, m_attributes)); + m_sink.write(m_level, formatLog("")); else - m_sink.write(m_level, formatLog(s, m_level, m_attributes)); + m_sink.write(m_level, formatLog(s)); } } @@ -387,7 +384,7 @@ Journal::ScopedStream::operator<<(std::ostream& manip(std::ostream&)) const Journal::ScopedStream Journal::Stream::operator<<(std::ostream& manip(std::ostream&)) const { - return {m_attributes, *this, manip}; + return {*this, manip}; } } // namespace beast From 61ff2ba0e7f27b33f6f70434c2ce39cbd9a88d16 Mon Sep 17 00:00:00 2001 From: JCW Date: Fri, 29 Aug 2025 22:47:26 +0100 Subject: [PATCH 25/88] Fix issues Signed-off-by: JCW --- include/xrpl/basics/Log.h | 4 ++-- include/xrpl/beast/utility/Journal.h | 2 +- src/libxrpl/basics/Log.cpp | 10 +++++----- src/libxrpl/beast/utility/beast_Journal.cpp | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/include/xrpl/basics/Log.h b/include/xrpl/basics/Log.h index 5ba31f89f0c..3b801f4bebe 100644 --- a/include/xrpl/basics/Log.h +++ b/include/xrpl/basics/Log.h @@ -206,7 +206,7 @@ class Logs write( beast::severities::Severity level, std::string const& partition, - std::string text, + std::string const& text, bool console); std::string @@ -244,7 +244,7 @@ class Logs static void format( std::string& output, - std::string message, + std::string const& message, beast::severities::Severity severity, std::string const& partition); diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index f54b57815ef..9e73f279dde 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -97,7 +97,7 @@ enum Severity { kNone = kDisabled }; -std::string +std::string_view to_string(Severity severity); } // namespace severities diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index a87d9bb315c..a699183221b 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -200,11 +200,11 @@ void Logs::write( beast::severities::Severity level, std::string const& partition, - std::string text, + std::string const& text, bool console) { std::string s; - format(s, std::move(text), level, partition); + format(s, text, level, partition); file_.writeln(s); if (!silent_) std::cerr << s << '\n'; @@ -332,14 +332,14 @@ Logs::fromString(std::string const& s) void Logs::format( std::string& output, - std::string message, + std::string const& message, beast::severities::Severity severity, std::string const& partition) { - output = std::move(message); + output = message; if (!beast::Journal::isStructuredJournalEnabled()) { - output.reserve(message.size() + partition.size() + 100); + output.reserve(output.size() + partition.size() + 100); output += to_string(std::chrono::system_clock::now()); output += " "; diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index 04e286b8db2..1c5f7ed7df6 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -98,7 +98,7 @@ Journal::getNullSink() //------------------------------------------------------------------------------ -std::string +std::string_view severities::to_string(Severity severity) { switch (severity) @@ -185,7 +185,7 @@ Journal::JsonLogContext::reset( value = threadIdStream.str(); } }; - thread_local ThreadIdStringInitializer threadId; + thread_local ThreadIdStringInitializer const threadId; attributes_.SetObject(); if (globalLogAttributes_.has_value()) From 48cf042258e5b814410d1d416367c9754c7f1ab3 Mon Sep 17 00:00:00 2001 From: JCW Date: Fri, 29 Aug 2025 22:59:53 +0100 Subject: [PATCH 26/88] Optimisation Signed-off-by: JCW --- include/xrpl/basics/Log.h | 4 ++-- include/xrpl/beast/utility/Journal.h | 6 +++--- include/xrpl/beast/utility/WrappedSink.h | 4 ++-- src/libxrpl/basics/Log.cpp | 8 ++++---- src/libxrpl/beast/utility/beast_Journal.cpp | 10 +++++----- src/test/beast/beast_Journal_test.cpp | 4 ++-- src/test/csf/Sim.h | 4 ++-- src/test/jtx/CaptureLogs.h | 4 ++-- src/test/jtx/CheckMessageLogs.h | 6 +++--- src/test/server/Server_test.cpp | 4 ++-- src/test/unit_test/SuiteJournal.h | 16 ++++++++-------- src/tests/libxrpl/basics/log.cpp | 8 ++++---- 12 files changed, 39 insertions(+), 39 deletions(-) diff --git a/include/xrpl/basics/Log.h b/include/xrpl/basics/Log.h index 3b801f4bebe..fe61f2c5413 100644 --- a/include/xrpl/basics/Log.h +++ b/include/xrpl/basics/Log.h @@ -68,11 +68,11 @@ class Logs operator=(Sink const&) = delete; void - write(beast::severities::Severity level, std::string const& text) + write(beast::severities::Severity level, std::string&& text) override; void - writeAlways(beast::severities::Severity level, std::string const& text) + writeAlways(beast::severities::Severity level, std::string&& text) override; }; diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 9e73f279dde..d501ae29f65 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -225,7 +225,7 @@ class Journal severities::Severity severity) const; static std::string - formatLog(std::string const& message); + formatLog(std::string&& message); public: //-------------------------------------------------------------------------- @@ -279,7 +279,7 @@ class Journal level is below the current threshold(). */ virtual void - write(Severity level, std::string const& text) = 0; + write(Severity level, std::string&& text) = 0; /** Bypass filter and write text to the sink at the specified severity. * Always write the message, but maintain the same formatting as if @@ -289,7 +289,7 @@ class Journal * @param text Text to write to sink. */ virtual void - writeAlways(Severity level, std::string const& text) = 0; + writeAlways(Severity level, std::string&& text) = 0; private: Severity thresh_; diff --git a/include/xrpl/beast/utility/WrappedSink.h b/include/xrpl/beast/utility/WrappedSink.h index 72dcf4e8f59..ee8641849d3 100644 --- a/include/xrpl/beast/utility/WrappedSink.h +++ b/include/xrpl/beast/utility/WrappedSink.h @@ -88,14 +88,14 @@ class WrappedSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, std::string const& text) override + write(beast::severities::Severity level, std::string&& text) override { using beast::Journal; sink_.write(level, prefix_ + text); } void - writeAlways(severities::Severity level, std::string const& text) override + writeAlways(severities::Severity level, std::string&& text) override { using beast::Journal; sink_.writeAlways(level, prefix_ + text); diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index a699183221b..60d4fadb8bb 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -47,20 +47,20 @@ Logs::Sink::Sink( } void -Logs::Sink::write(beast::severities::Severity level, std::string const& text) +Logs::Sink::write(beast::severities::Severity level, std::string&& text) { if (level < threshold()) return; - logs_.write(level, partition_, text, console()); + logs_.write(level, partition_, std::move(text), console()); } void Logs::Sink::writeAlways( beast::severities::Severity level, - std::string const& text) + std::string&& text) { - logs_.write(level, partition_, text, console()); + logs_.write(level, partition_, std::move(text), console()); } //------------------------------------------------------------------------------ diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index 1c5f7ed7df6..6aa617722dc 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -77,12 +77,12 @@ class NullJournalSink : public Journal::Sink } void - write(severities::Severity, std::string const&) override + write(severities::Severity, std::string&&) override { } void - writeAlways(severities::Severity, std::string const&) override + writeAlways(severities::Severity, std::string&&) override { } }; @@ -264,7 +264,7 @@ Journal::initMessageContext( } std::string -Journal::formatLog(std::string const& message) +Journal::formatLog(std::string&& message) { if (!m_jsonLogsEnabled) { @@ -363,13 +363,13 @@ Journal::ScopedStream::ScopedStream( Journal::ScopedStream::~ScopedStream() { - std::string const& s(m_ostream.str()); + std::string s(m_ostream.str()); if (!s.empty()) { if (s == "\n") m_sink.write(m_level, formatLog("")); else - m_sink.write(m_level, formatLog(s)); + m_sink.write(m_level, formatLog(std::move(s))); } } diff --git a/src/test/beast/beast_Journal_test.cpp b/src/test/beast/beast_Journal_test.cpp index 13e2726c89c..abd4af4f8dd 100644 --- a/src/test/beast/beast_Journal_test.cpp +++ b/src/test/beast/beast_Journal_test.cpp @@ -48,14 +48,14 @@ class Journal_test : public unit_test::suite } void - write(severities::Severity level, std::string const&) override + write(severities::Severity level, std::string&&) override { if (level >= threshold()) ++m_count; } void - writeAlways(severities::Severity level, std::string const&) override + writeAlways(severities::Severity level, std::string&&) override { ++m_count; } diff --git a/src/test/csf/Sim.h b/src/test/csf/Sim.h index 0cb052305ad..cf4725c7098 100644 --- a/src/test/csf/Sim.h +++ b/src/test/csf/Sim.h @@ -49,7 +49,7 @@ class BasicSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, std::string const& text) override + write(beast::severities::Severity level, std::string&& text) override { if (level < threshold()) return; @@ -59,7 +59,7 @@ class BasicSink : public beast::Journal::Sink } void - writeAlways(beast::severities::Severity level, std::string const& text) + writeAlways(beast::severities::Severity level, std::string&& text) override { std::cout << clock_.now().time_since_epoch().count() << " " << text diff --git a/src/test/jtx/CaptureLogs.h b/src/test/jtx/CaptureLogs.h index a8afb521c38..20bf4dd51aa 100644 --- a/src/test/jtx/CaptureLogs.h +++ b/src/test/jtx/CaptureLogs.h @@ -57,7 +57,7 @@ class CaptureLogs : public Logs } void - write(beast::severities::Severity level, std::string const& text) + write(beast::severities::Severity level, std::string&& text) override { std::lock_guard lock(strmMutex_); @@ -65,7 +65,7 @@ class CaptureLogs : public Logs } void - writeAlways(beast::severities::Severity level, std::string const& text) + writeAlways(beast::severities::Severity level, std::string&& text) override { std::lock_guard lock(strmMutex_); diff --git a/src/test/jtx/CheckMessageLogs.h b/src/test/jtx/CheckMessageLogs.h index fd3915440e1..12673a8a142 100644 --- a/src/test/jtx/CheckMessageLogs.h +++ b/src/test/jtx/CheckMessageLogs.h @@ -45,7 +45,7 @@ class CheckMessageLogs : public Logs } void - write(beast::severities::Severity level, std::string const& text) + write(beast::severities::Severity level, std::string&& text) override { if (text.find(owner_.msg_) != std::string::npos) @@ -53,10 +53,10 @@ class CheckMessageLogs : public Logs } void - writeAlways(beast::severities::Severity level, std::string const& text) + writeAlways(beast::severities::Severity level, std::string&& text) override { - write(level, text); + write(level, std::move(text)); } }; diff --git a/src/test/server/Server_test.cpp b/src/test/server/Server_test.cpp index 874558f428b..08b18312573 100644 --- a/src/test/server/Server_test.cpp +++ b/src/test/server/Server_test.cpp @@ -92,7 +92,7 @@ class Server_test : public beast::unit_test::suite } void - write(beast::severities::Severity level, std::string const& text) + write(beast::severities::Severity level, std::string&& text) override { if (level < threshold()) @@ -102,7 +102,7 @@ class Server_test : public beast::unit_test::suite } void - writeAlways(beast::severities::Severity level, std::string const& text) + writeAlways(beast::severities::Severity level, std::string&& text) override { suite_.log << text << std::endl; diff --git a/src/test/unit_test/SuiteJournal.h b/src/test/unit_test/SuiteJournal.h index d56c297b0a2..f0421080b6a 100644 --- a/src/test/unit_test/SuiteJournal.h +++ b/src/test/unit_test/SuiteJournal.h @@ -49,27 +49,27 @@ class SuiteJournalSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, std::string const& text) override; + write(beast::severities::Severity level, std::string&& text) override; void - writeAlways(beast::severities::Severity level, std::string const& text) + writeAlways(beast::severities::Severity level, std::string&& text) override; }; inline void SuiteJournalSink::write( beast::severities::Severity level, - std::string const& text) + std::string&& text) { // Only write the string if the level at least equals the threshold. if (level >= threshold()) - writeAlways(level, text); + writeAlways(level, std::move(text)); } inline void SuiteJournalSink::writeAlways( beast::severities::Severity level, - std::string const& text) + std::string&& text) { using namespace beast::severities; @@ -137,15 +137,15 @@ class StreamSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, std::string const& text) override + write(beast::severities::Severity level, std::string&& text) override { if (level < threshold()) return; - writeAlways(level, text); + writeAlways(level, std::move(text)); } inline void - writeAlways(beast::severities::Severity level, std::string const& text) + writeAlways(beast::severities::Severity level, std::string&& text) override { strm_ << text << std::endl; diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index fb634a3a71b..de5a2aa5a24 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -49,14 +49,14 @@ class MockLogs : public Logs operator=(Sink const&) = delete; void - write(beast::severities::Severity level, std::string const& text) + write(beast::severities::Severity level, std::string&& text) override { logs_.logStream_ << text; } void - writeAlways(beast::severities::Severity level, std::string const& text) + writeAlways(beast::severities::Severity level, std::string&& text) override { logs_.logStream_ << text; @@ -217,13 +217,13 @@ class MockSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, std::string const& text) override + write(beast::severities::Severity level, std::string&& text) override { strm_ << text; } void - writeAlways(beast::severities::Severity level, std::string const& text) + writeAlways(beast::severities::Severity level, std::string&& text) override { strm_ << text; From 157aa367f24aaf56b18638adedd660c77ff11980 Mon Sep 17 00:00:00 2001 From: JCW Date: Fri, 29 Aug 2025 23:06:24 +0100 Subject: [PATCH 27/88] Fix issues Signed-off-by: JCW --- src/libxrpl/beast/utility/beast_Journal.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index 6aa617722dc..d432d8b7e08 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -363,7 +363,7 @@ Journal::ScopedStream::ScopedStream( Journal::ScopedStream::~ScopedStream() { - std::string s(m_ostream.str()); + std::string s = m_ostream.str(); if (!s.empty()) { if (s == "\n") From 3181042f15b4bd4fe829667738e1edd1a43041e2 Mon Sep 17 00:00:00 2001 From: JCW Date: Sat, 30 Aug 2025 02:03:02 +0100 Subject: [PATCH 28/88] Fix issues Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index d501ae29f65..89a046892b1 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -458,17 +458,15 @@ class Journal Journal( Journal const& other, std::optional attributes = std::nullopt) - : m_sink(other.m_sink) + : m_attributes(other.m_attributes) + , m_sink(other.m_sink) { if (attributes.has_value()) - m_attributes = std::move(attributes.value()); - if (other.m_attributes.has_value()) { - if (m_attributes.has_value()) - m_attributes->combine( - other.m_attributes->contextValues_); + if (m_attributes) + m_attributes->combine(attributes->contextValues_); else - m_attributes = other.m_attributes; + m_attributes = std::move(attributes); } } /** Create a journal that writes to the specified sink. */ From 7500d635bb7612f8628175e76bf4116194a7aa48 Mon Sep 17 00:00:00 2001 From: JCW Date: Sat, 30 Aug 2025 20:59:52 +0100 Subject: [PATCH 29/88] Fix issues Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 2 +- src/libxrpl/beast/utility/beast_Journal.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 89a046892b1..10715ad93ae 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -97,7 +97,7 @@ enum Severity { kNone = kDisabled }; -std::string_view +std::string to_string(Severity severity); } // namespace severities diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index d432d8b7e08..65593db2140 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -98,7 +98,7 @@ Journal::getNullSink() //------------------------------------------------------------------------------ -std::string_view +std::string severities::to_string(Severity severity) { switch (severity) From dca000a60f67f00a136e44f05a27d419440b575d Mon Sep 17 00:00:00 2001 From: JCW Date: Sun, 31 Aug 2025 14:28:26 +0100 Subject: [PATCH 30/88] Optimisation Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 47 +++---- src/libxrpl/beast/utility/beast_Journal.cpp | 147 +++++++++++++------- src/tests/libxrpl/basics/log.cpp | 140 ++++++++++--------- 3 files changed, 187 insertions(+), 147 deletions(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 10715ad93ae..4e4bd3ae3fd 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -174,8 +174,9 @@ class Journal class JsonLogContext { - rapidjson::Value attributes_; + rapidjson::Value messageParams_; rapidjson::MemoryPoolAllocator<> allocator_; + std::optional journalAttributesJson_; public: JsonLogContext() = default; @@ -189,20 +190,20 @@ class Journal rapidjson::Value& messageParams() { - return attributes_["Params"]; + return messageParams_; } - rapidjson::Value& - attributes() + std::optional& + journalAttributesJson() { - return attributes_; + return journalAttributesJson_; } void reset( std::source_location location, severities::Severity severity, - std::optional const& attributes) noexcept; + std::optional const& journalAttributesJson) noexcept; }; private: @@ -210,7 +211,9 @@ class Journal using Severity = severities::Severity; std::optional m_attributes; + std::optional m_attributesJson; static std::optional globalLogAttributes_; + static std::optional globalLogAttributesJson_; static std::mutex globalLogAttributesMutex_; static bool m_jsonLogsEnabled; @@ -227,6 +230,9 @@ class Journal static std::string formatLog(std::string&& message); + void + rebuildAttributeJson(); + public: //-------------------------------------------------------------------------- @@ -457,18 +463,8 @@ class Journal Journal( Journal const& other, - std::optional attributes = std::nullopt) - : m_attributes(other.m_attributes) - , m_sink(other.m_sink) - { - if (attributes.has_value()) - { - if (m_attributes) - m_attributes->combine(attributes->contextValues_); - else - m_attributes = std::move(attributes); - } - } + std::optional attributes = std::nullopt); + /** Create a journal that writes to the specified sink. */ explicit Journal( Sink& sink, @@ -481,6 +477,7 @@ class Journal m_attributes = std::move(attributes); m_attributes->setModuleName(name); } + rebuildAttributeJson(); } Journal& @@ -491,6 +488,7 @@ class Journal m_sink = other.m_sink; m_attributes = other.m_attributes; + rebuildAttributeJson(); return *this; } @@ -499,6 +497,7 @@ class Journal { m_sink = other.m_sink; m_attributes = std::move(other.m_attributes); + rebuildAttributeJson(); return *this; } @@ -586,19 +585,11 @@ class Journal { std::lock_guard lock(globalLogAttributesMutex_); globalLogAttributes_ = std::nullopt; + globalLogAttributesJson_ = std::nullopt; } static void - addGlobalAttributes(JsonLogAttributes globalLogAttributes) - { - std::lock_guard lock(globalLogAttributesMutex_); - if (!globalLogAttributes_) - { - globalLogAttributes_ = JsonLogAttributes{}; - } - globalLogAttributes_->combine( - globalLogAttributes.contextValues()); - } + addGlobalAttributes(JsonLogAttributes globalLogAttributes); }; #ifndef __INTELLISENSE__ diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index 65593db2140..b523477cf7f 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -32,6 +32,8 @@ namespace beast { std::optional Journal::globalLogAttributes_; +std::optional Journal::globalLogAttributesJson_; + std::mutex Journal::globalLogAttributesMutex_; bool Journal::m_jsonLogsEnabled = false; thread_local Journal::JsonLogContext Journal::currentJsonLogContext_{}; @@ -169,11 +171,44 @@ Journal::JsonLogAttributes::combine( } } +Journal::Journal( + Journal const& other, + std::optional attributes) + : m_attributes(other.m_attributes), m_sink(other.m_sink) +{ + if (attributes.has_value()) + { + if (m_attributes) + m_attributes->combine(attributes->contextValues_); + else + m_attributes = std::move(attributes); + } + rebuildAttributeJson(); +} + +void +Journal::addGlobalAttributes(JsonLogAttributes globalLogAttributes) +{ + std::lock_guard lock(globalLogAttributesMutex_); + if (!globalLogAttributes_) + { + globalLogAttributes_ = JsonLogAttributes{}; + } + globalLogAttributes_->combine(globalLogAttributes.contextValues()); + + rapidjson::StringBuffer buffer; + rapidjson::Writer writer{buffer}; + + globalLogAttributes_->contextValues().Accept(writer); + + globalLogAttributesJson_ = {buffer.GetString()}; +} + void Journal::JsonLogContext::reset( std::source_location location, severities::Severity severity, - std::optional const& attributes) noexcept + std::optional const& journalAttributesJson) noexcept { struct ThreadIdStringInitializer { @@ -187,67 +222,37 @@ Journal::JsonLogContext::reset( }; thread_local ThreadIdStringInitializer const threadId; - attributes_.SetObject(); - if (globalLogAttributes_.has_value()) - { - attributes_.CopyFrom(globalLogAttributes_->contextValues(), allocator_); - if (attributes.has_value()) - { - for (auto const& [key, value] : - attributes->contextValues().GetObject()) - { - attributes_.RemoveMember(key); - - rapidjson::Value jsonValue; - jsonValue.CopyFrom(value, allocator_); - - attributes_.AddMember( - rapidjson::Value{key, allocator_}, - rapidjson::Value{value, allocator_}, - allocator_); - } - } - } - else if (attributes.has_value()) - { - attributes_.CopyFrom(attributes->contextValues(), allocator_); - } + journalAttributesJson_ = journalAttributesJson; + + messageParams_.SetObject(); - attributes_.RemoveMember("Function"); - attributes_.AddMember( + messageParams_.AddMember( rapidjson::StringRef("Function"), rapidjson::Value{location.function_name(), allocator_}, allocator_); - attributes_.RemoveMember("File"); - attributes_.AddMember( + messageParams_.AddMember( rapidjson::StringRef("File"), rapidjson::Value{location.file_name(), allocator_}, allocator_); - attributes_.RemoveMember("Line"); - attributes_.AddMember( + messageParams_.AddMember( rapidjson::StringRef("Line"), location.line(), allocator_); - attributes_.RemoveMember("ThreadId"); - attributes_.AddMember( + + messageParams_.AddMember( rapidjson::StringRef("ThreadId"), rapidjson::Value{threadId.value.c_str(), allocator_}, allocator_); - attributes_.RemoveMember("Params"); - attributes_.AddMember( - rapidjson::StringRef("Params"), - rapidjson::Value{rapidjson::kObjectType}, - allocator_); + auto severityStr = to_string(severity); - attributes_.RemoveMember("Level"); - attributes_.AddMember( + messageParams_.AddMember( rapidjson::StringRef("Level"), rapidjson::Value{severityStr.c_str(), allocator_}, allocator_); - attributes_.RemoveMember("Time"); - attributes_.AddMember( + + messageParams_.AddMember( rapidjson::StringRef("Time"), std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) @@ -260,7 +265,7 @@ Journal::initMessageContext( std::source_location location, severities::Severity severity) const { - currentJsonLogContext_.reset(location, severity, m_attributes); + currentJsonLogContext_.reset(location, severity, m_attributesJson); } std::string @@ -271,23 +276,59 @@ Journal::formatLog(std::string&& message) return message; } - auto& attributes = currentJsonLogContext_.attributes(); - - attributes.RemoveMember("Message"); - attributes.AddMember( - rapidjson::StringRef("Message"), - rapidjson::Value{rapidjson::StringRef(message.c_str()), currentJsonLogContext_.allocator()}, - currentJsonLogContext_.allocator() - ); + auto& messageParams = currentJsonLogContext_.messageParams(); rapidjson::StringBuffer buffer; rapidjson::Writer writer(buffer); - attributes.Accept(writer); + writer.StartObject(); + if (globalLogAttributesJson_.has_value()) + { + writer.Key("GlobalParams"); + writer.RawValue( + globalLogAttributesJson_->c_str(), + globalLogAttributesJson_->length(), + rapidjson::kObjectType); + } + + if (currentJsonLogContext_.journalAttributesJson().has_value()) + { + writer.Key("JournalParams"); + writer.RawValue( + currentJsonLogContext_.journalAttributesJson()->c_str(), + currentJsonLogContext_.journalAttributesJson()->length(), + rapidjson::kObjectType); + } + + writer.Key("MessageParams"); + messageParams.Accept(writer); + + writer.Key("Message"); + writer.String(message.c_str(), message.length()); + + writer.EndObject(); return {buffer.GetString()}; } +void +Journal::rebuildAttributeJson() +{ + if (m_attributes.has_value()) + { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer{buffer}; + + m_attributes->contextValues().Accept(writer); + + m_attributesJson = {buffer.GetString()}; + } + else + { + m_attributesJson = {}; + } +} + void Journal::enableStructuredJournal() { diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index de5a2aa5a24..4c2b14eb988 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -164,9 +164,11 @@ TEST_CASE("Global attributes") jsonLog.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); CHECK(jsonLog.IsObject()); - CHECK(jsonLog.HasMember("Field1")); - CHECK(jsonLog["Field1"].IsString()); - CHECK(jsonLog["Field1"].GetString() == std::string{"Value1"}); + CHECK(jsonLog.HasMember("GlobalParams")); + CHECK(jsonLog["GlobalParams"].IsObject()); + CHECK(jsonLog["GlobalParams"].HasMember("Field1")); + CHECK(jsonLog["GlobalParams"]["Field1"].IsString()); + CHECK(jsonLog["GlobalParams"]["Field1"].GetString() == std::string{"Value1"}); beast::Journal::disableStructuredJournal(); } @@ -194,12 +196,11 @@ TEST_CASE("Global attributes inheritable") jsonLog.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); CHECK(jsonLog.IsObject()); - CHECK(jsonLog.HasMember("Field1")); - CHECK(jsonLog["Field1"].IsString()); - // Field1 should be overwritten to Value3 - CHECK(jsonLog["Field1"].GetString() == std::string{"Value3"}); - CHECK(jsonLog["Field2"].IsString()); - CHECK(jsonLog["Field2"].GetString() == std::string{"Value2"}); + CHECK(jsonLog["GlobalParams"].HasMember("Field1")); + CHECK(jsonLog["GlobalParams"]["Field1"].IsString()); + CHECK(jsonLog["GlobalParams"]["Field1"].GetString() == std::string{"Value1"}); + CHECK(jsonLog["JournalParams"]["Field1"].GetString() == std::string{"Value3"}); + CHECK(jsonLog["JournalParams"]["Field2"].GetString() == std::string{"Value2"}); beast::Journal::disableStructuredJournal(); } @@ -234,7 +235,9 @@ class JsonLogStreamFixture { public: JsonLogStreamFixture() - : sink_(beast::severities::kAll, logStream_), j_(sink_) + : sink_(beast::severities::kAll, logStream_), j_(sink_, "Test", log::attributes( + log::attr("Field1", "Value1") + )) { beast::Journal::enableStructuredJournal(); } @@ -264,8 +267,9 @@ class JsonLogStreamFixture TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") { + beast::Journal::addGlobalAttributes(log::attributes(log::attr("Field2", "Value2"))); journal().debug() << std::boolalpha << true << std::noboolalpha << " Test " - << std::boolalpha << false; + << std::boolalpha << false << log::field("Field3", "Value3"); rapidjson::Document logValue; logValue.Parse(stream().str().c_str()); @@ -274,20 +278,27 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); CHECK(logValue.IsObject()); - CHECK(logValue.HasMember("Function")); - CHECK(logValue.HasMember("File")); - CHECK(logValue.HasMember("Line")); - CHECK(logValue.HasMember("ThreadId")); - CHECK(logValue.HasMember("Params")); - CHECK(logValue.HasMember("Level")); + CHECK(logValue.HasMember("GlobalParams")); + CHECK(logValue.HasMember("JournalParams")); + CHECK(logValue.HasMember("MessageParams")); CHECK(logValue.HasMember("Message")); - CHECK(logValue.HasMember("Time")); - CHECK(logValue["Function"].IsString()); - CHECK(logValue["File"].IsString()); - CHECK(logValue["Line"].IsNumber()); - CHECK(logValue["Params"].IsObject()); + CHECK(logValue["GlobalParams"].IsObject()); + CHECK(logValue["JournalParams"].IsObject()); + CHECK(logValue["MessageParams"].IsObject()); CHECK(logValue["Message"].IsString()); + + CHECK(logValue["MessageParams"].HasMember("Function")); + CHECK(logValue["MessageParams"].HasMember("File")); + CHECK(logValue["MessageParams"].HasMember("Line")); + CHECK(logValue["MessageParams"].HasMember("ThreadId")); + CHECK(logValue["MessageParams"].HasMember("Level")); + CHECK(logValue["MessageParams"].HasMember("Time")); + + CHECK(logValue["MessageParams"]["Function"].IsString()); + CHECK(logValue["MessageParams"]["File"].IsString()); + CHECK(logValue["MessageParams"]["Line"].IsNumber()); + CHECK(logValue["Message"].GetString() == std::string{"true Test false"}); } @@ -305,7 +316,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") rapidjson::ParseErrorCode::kParseErrorNone); CHECK( - logValue["Level"].GetString() == + logValue["MessageParams"]["Level"].GetString() == beast::severities::to_string(beast::severities::kTrace)); } @@ -321,7 +332,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") rapidjson::ParseErrorCode::kParseErrorNone); CHECK( - logValue["Level"].GetString() == + logValue["MessageParams"]["Level"].GetString() == beast::severities::to_string(beast::severities::kDebug)); } @@ -337,7 +348,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") rapidjson::ParseErrorCode::kParseErrorNone); CHECK( - logValue["Level"].GetString() == + logValue["MessageParams"]["Level"].GetString() == beast::severities::to_string(beast::severities::kInfo)); } @@ -353,7 +364,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") rapidjson::ParseErrorCode::kParseErrorNone); CHECK( - logValue["Level"].GetString() == + logValue["MessageParams"]["Level"].GetString() == beast::severities::to_string(beast::severities::kWarning)); } @@ -369,7 +380,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") rapidjson::ParseErrorCode::kParseErrorNone); CHECK( - logValue["Level"].GetString() == + logValue["MessageParams"]["Level"].GetString() == beast::severities::to_string(beast::severities::kError)); } @@ -385,7 +396,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") rapidjson::ParseErrorCode::kParseErrorNone); CHECK( - logValue["Level"].GetString() == + logValue["MessageParams"]["Level"].GetString() == beast::severities::to_string(beast::severities::kFatal)); } } @@ -401,7 +412,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogStream") logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); CHECK( - logValue["Level"].GetString() == + logValue["MessageParams"]["Level"].GetString() == beast::severities::to_string(beast::severities::kError)); } @@ -418,15 +429,12 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogParams") CHECK( logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); - CHECK(logValue["Params"].IsObject()); - CHECK(logValue["Params"]["Field1"].IsNumber()); - CHECK(logValue["Params"]["Field1"].GetInt() == 1); - // UInt64 doesn't fit in Json::Value so it should be converted to a string - // NOTE: We should expect it to be an int64 after we make the json library - // support in64 and uint64 - CHECK(logValue["Params"]["Field2"].IsNumber()); + CHECK(logValue["MessageParams"].IsObject()); + CHECK(logValue["MessageParams"]["Field1"].IsNumber()); + CHECK(logValue["MessageParams"]["Field1"].GetInt() == 1); + CHECK(logValue["MessageParams"]["Field2"].IsNumber()); CHECK( - logValue["Params"]["Field2"].GetUint64() == + logValue["MessageParams"]["Field2"].GetUint64() == std::numeric_limits::max()); CHECK(logValue["Message"].IsString()); CHECK( @@ -447,15 +455,15 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") CHECK( logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); - CHECK(logValue["Params"].IsObject()); - CHECK(logValue["Params"]["Field1"].IsNumber()); - CHECK(logValue["Params"]["Field1"].GetInt() == 1); + CHECK(logValue["MessageParams"].IsObject()); + CHECK(logValue["MessageParams"]["Field1"].IsNumber()); + CHECK(logValue["MessageParams"]["Field1"].GetInt() == 1); // UInt64 doesn't fit in Json::Value so it should be converted to a string // NOTE: We should expect it to be an int64 after we make the json library // support in64 and uint64 - CHECK(logValue["Params"]["Field2"].IsNumber()); + CHECK(logValue["MessageParams"]["Field2"].IsNumber()); CHECK( - logValue["Params"]["Field2"].GetUint64() == + logValue["MessageParams"]["Field2"].GetUint64() == std::numeric_limits::max()); CHECK(logValue["Message"].IsString()); CHECK(logValue["Message"].GetString() == std::string{"Test"}); @@ -475,10 +483,10 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributes") CHECK( logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); - CHECK(logValue["Field1"].IsString()); - CHECK(logValue["Field1"].GetString() == std::string{"Value1"}); - CHECK(logValue["Field2"].IsNumber()); - CHECK(logValue["Field2"].GetInt() == 2); + CHECK(logValue["JournalParams"]["Field1"].IsString()); + CHECK(logValue["JournalParams"]["Field1"].GetString() == std::string{"Value1"}); + CHECK(logValue["JournalParams"]["Field2"].IsNumber()); + CHECK(logValue["JournalParams"]["Field2"].GetInt() == 2); } TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributesInheritable") @@ -498,13 +506,13 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributesInheritable") CHECK( logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); - CHECK(logValue["Field1"].IsString()); - CHECK(logValue["Field1"].GetString() == std::string{"Value1"}); - CHECK(logValue["Field3"].IsString()); - CHECK(logValue["Field3"].GetString() == std::string{"Value3"}); + CHECK(logValue["JournalParams"]["Field1"].IsString()); + CHECK(logValue["JournalParams"]["Field1"].GetString() == std::string{"Value1"}); + CHECK(logValue["JournalParams"]["Field3"].IsString()); + CHECK(logValue["JournalParams"]["Field3"].GetString() == std::string{"Value3"}); // Field2 should be overwritten to 0 - CHECK(logValue["Field2"].IsNumber()); - CHECK(logValue["Field2"].GetInt() == 0); + CHECK(logValue["JournalParams"]["Field2"].IsNumber()); + CHECK(logValue["JournalParams"]["Field2"].GetInt() == 0); } TEST_CASE_FIXTURE( @@ -526,13 +534,13 @@ TEST_CASE_FIXTURE( CHECK( logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); - CHECK(logValue["Field1"].IsString()); - CHECK(logValue["Field1"].GetString() == std::string{"Value1"}); - CHECK(logValue["Field3"].IsString()); - CHECK(logValue["Field3"].GetString() == std::string{"Value3"}); + CHECK(logValue["JournalParams"]["Field1"].IsString()); + CHECK(logValue["JournalParams"]["Field1"].GetString() == std::string{"Value1"}); + CHECK(logValue["JournalParams"]["Field3"].IsString()); + CHECK(logValue["JournalParams"]["Field3"].GetString() == std::string{"Value3"}); // Field2 should be overwritten to 0 - CHECK(logValue["Field2"].IsNumber()); - CHECK(logValue["Field2"].GetInt() == 0); + CHECK(logValue["JournalParams"]["Field2"].IsNumber()); + CHECK(logValue["JournalParams"]["Field2"].GetInt() == 0); } TEST_CASE_FIXTURE( @@ -555,10 +563,10 @@ TEST_CASE_FIXTURE( CHECK( logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); - CHECK(logValue["Field1"].IsString()); - CHECK(logValue["Field1"].GetString() == std::string{"Value1"}); - CHECK(logValue["Field2"].IsNumber()); - CHECK(logValue["Field2"].GetInt() == 2); + CHECK(logValue["JournalParams"]["Field1"].IsString()); + CHECK(logValue["JournalParams"]["Field1"].GetString() == std::string{"Value1"}); + CHECK(logValue["JournalParams"]["Field2"].IsNumber()); + CHECK(logValue["JournalParams"]["Field2"].GetInt() == 2); } TEST_CASE_FIXTURE( @@ -581,8 +589,8 @@ TEST_CASE_FIXTURE( CHECK( logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); - CHECK(logValue["Field1"].IsString()); - CHECK(logValue["Field1"].GetString() == std::string{"Value1"}); - CHECK(logValue["Field2"].IsNumber()); - CHECK(logValue["Field2"].GetInt() == 2); + CHECK(logValue["JournalParams"]["Field1"].IsString()); + CHECK(logValue["JournalParams"]["Field1"].GetString() == std::string{"Value1"}); + CHECK(logValue["JournalParams"]["Field2"].IsNumber()); + CHECK(logValue["JournalParams"]["Field2"].GetInt() == 2); } \ No newline at end of file From 5e060a9e7bbbd0bc1d5d085f8642bd786886cf78 Mon Sep 17 00:00:00 2001 From: JCW Date: Mon, 1 Sep 2025 11:59:50 +0100 Subject: [PATCH 31/88] Fix Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 126 +++++++++++++------- src/libxrpl/beast/utility/beast_Journal.cpp | 62 +++------- 2 files changed, 102 insertions(+), 86 deletions(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 4e4bd3ae3fd..c3e0012aef9 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -22,7 +22,13 @@ #include +#include +#include +#include + #include +#include +#include #include #include @@ -174,23 +180,27 @@ class Journal class JsonLogContext { - rapidjson::Value messageParams_; - rapidjson::MemoryPoolAllocator<> allocator_; + rapidjson::StringBuffer buffer_; + rapidjson::Writer messageParamsWriter_; std::optional journalAttributesJson_; public: - JsonLogContext() = default; + JsonLogContext() + : messageParamsWriter_(buffer_) + { - rapidjson::MemoryPoolAllocator<>& - allocator() + } + + rapidjson::Writer& + writer() { - return allocator_; + return messageParamsWriter_; } - rapidjson::Value& + char const* messageParams() { - return messageParams_; + return buffer_.GetString(); } std::optional& @@ -461,9 +471,28 @@ class Journal /** Journal has no default constructor. */ Journal() = delete; + template Journal( Journal const& other, - std::optional attributes = std::nullopt); + TAttributesFactory&& attributesFactory = nullptr) + : m_attributes(other.m_attributes) + , m_sink(other.m_sink) + , m_attributesJson(other.m_attributesJson) + { +/* + if constexpr (!std::is_same_v, std::nullptr_t>) + { + if (attributes.has_value()) + { + if (m_attributes) + m_attributes->combine(attributes->contextValues_); + else + m_attributes = std::move(attributes); + } + rebuildAttributeJson(); + } +*/ + } /** Create a journal that writes to the specified sink. */ explicit Journal( @@ -699,39 +728,59 @@ using logwstream = basic_logstream; namespace ripple::log { namespace detail { -template +template void setJsonValue( - rapidjson::Value& object, - rapidjson::MemoryPoolAllocator<>& allocator, + rapidjson::Writer& writer, char const* name, T&& value, std::ostream* outStream) { using ValueType = std::decay_t; - rapidjson::Value jsonValue; - if constexpr (std::constructible_from< - rapidjson::Value, - ValueType, - rapidjson::MemoryPoolAllocator<>&>) + writer.Key(name); + if constexpr (std::is_integral_v) + { + if constexpr (std::is_signed_v) + { + writer.Int64(value); + } + else + { + writer.Uint64(value); + } + if (outStream) + { + (*outStream) << value; + } + } + else if constexpr (std::is_floating_point_v) + { + writer.Double(value); + + if (outStream) + { + (*outStream) << value; + } + } + else if constexpr (std::is_same_v) { - jsonValue = rapidjson::Value{value, allocator}; + writer.Bool(value); if (outStream) { (*outStream) << value; } } - else if constexpr (std::constructible_from) + else if constexpr (std::is_same_v || std::is_same_v) { - jsonValue = rapidjson::Value{value}; + writer.String(value); if (outStream) { (*outStream) << value; } } - else if constexpr (std::same_as) + else if constexpr (std::is_same_v) { - jsonValue = rapidjson::Value{value.c_str(), allocator}; + writer.String(value.c_str(), value.length()); if (outStream) { (*outStream) << value; @@ -742,17 +791,13 @@ setJsonValue( std::ostringstream oss; oss << value; - jsonValue = rapidjson::Value{oss.str().c_str(), allocator}; + writer.String(oss.str().c_str(), oss.str().length()); if (outStream) { (*outStream) << oss.str(); } } - - object.RemoveMember(name); - object.AddMember( - rapidjson::StringRef(name), std::move(jsonValue), allocator); } } // namespace detail @@ -763,8 +808,7 @@ operator<<(std::ostream& os, LogParameter const& param) if (!beast::Journal::m_jsonLogsEnabled) return os; detail::setJsonValue( - beast::Journal::currentJsonLogContext_.messageParams(), - beast::Journal::currentJsonLogContext_.allocator(), + beast::Journal::currentJsonLogContext_.writer(), param.name_, param.value_, &os); @@ -778,8 +822,7 @@ operator<<(std::ostream& os, LogField const& param) if (!beast::Journal::m_jsonLogsEnabled) return os; detail::setJsonValue( - beast::Journal::currentJsonLogContext_.messageParams(), - beast::Journal::currentJsonLogContext_.allocator(), + beast::Journal::currentJsonLogContext_.writer(), param.name_, param.value_, nullptr); @@ -801,20 +844,17 @@ field(char const* name, T&& value) } template -[[nodiscard]] beast::Journal::JsonLogAttributes +[[nodiscard]] auto attributes(Pair&&... pairs) { - beast::Journal::JsonLogAttributes result; - - (detail::setJsonValue( - result.contextValues(), - result.allocator(), - pairs.first, - pairs.second, - nullptr), - ...); - - return result; + return [&](rapidjson::Writer>& writer) { + (detail::setJsonValue( + writer, + pairs.first, + pairs.second, + nullptr), + ...); + }; } template diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index b523477cf7f..b900cd5e409 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -171,21 +171,6 @@ Journal::JsonLogAttributes::combine( } } -Journal::Journal( - Journal const& other, - std::optional attributes) - : m_attributes(other.m_attributes), m_sink(other.m_sink) -{ - if (attributes.has_value()) - { - if (m_attributes) - m_attributes->combine(attributes->contextValues_); - else - m_attributes = std::move(attributes); - } - rebuildAttributeJson(); -} - void Journal::addGlobalAttributes(JsonLogAttributes globalLogAttributes) { @@ -224,40 +209,30 @@ Journal::JsonLogContext::reset( journalAttributesJson_ = journalAttributesJson; - messageParams_.SetObject(); + buffer_.Clear(); - messageParams_.AddMember( - rapidjson::StringRef("Function"), - rapidjson::Value{location.function_name(), allocator_}, - allocator_); + writer().StartObject(); - messageParams_.AddMember( - rapidjson::StringRef("File"), - rapidjson::Value{location.file_name(), allocator_}, - allocator_); + writer().Key("Function"); + writer().String(location.function_name()); - messageParams_.AddMember( - rapidjson::StringRef("Line"), - location.line(), - allocator_); + writer().Key("File"); + writer().String(location.file_name()); - messageParams_.AddMember( - rapidjson::StringRef("ThreadId"), - rapidjson::Value{threadId.value.c_str(), allocator_}, - allocator_); + writer().Key("Line"); + writer().Int64(location.line()); + + writer().Key("ThreadId"); + writer().String(threadId.value.c_str()); auto severityStr = to_string(severity); - messageParams_.AddMember( - rapidjson::StringRef("Level"), - rapidjson::Value{severityStr.c_str(), allocator_}, - allocator_); + writer().Key("Level"); + writer().String(severityStr.c_str()); - messageParams_.AddMember( - rapidjson::StringRef("Time"), - std::chrono::duration_cast( + writer().Key("Time"); + writer().Uint64(std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) - .count(), - allocator_); + .count()); } void @@ -276,7 +251,8 @@ Journal::formatLog(std::string&& message) return message; } - auto& messageParams = currentJsonLogContext_.messageParams(); + currentJsonLogContext_.writer().EndObject(); + auto messageParams = currentJsonLogContext_.messageParams(); rapidjson::StringBuffer buffer; rapidjson::Writer writer(buffer); @@ -301,7 +277,7 @@ Journal::formatLog(std::string&& message) } writer.Key("MessageParams"); - messageParams.Accept(writer); + writer.RawValue(messageParams, std::strlen(messageParams), rapidjson::kObjectType); writer.Key("Message"); writer.String(message.c_str(), message.length()); From 90f970be46dbfa96f5ec076d157d4637a95600e0 Mon Sep 17 00:00:00 2001 From: JCW Date: Mon, 1 Sep 2025 18:15:20 +0100 Subject: [PATCH 32/88] Improve performance Signed-off-by: JCW --- include/xrpl/basics/Log.h | 19 +- include/xrpl/beast/utility/Journal.h | 308 +++++++++++--------- src/libxrpl/basics/Log.cpp | 12 +- src/libxrpl/beast/utility/beast_Journal.cpp | 177 +++-------- src/test/csf/Sim.h | 3 +- src/test/jtx/CaptureLogs.h | 3 +- src/test/jtx/CheckMessageLogs.h | 3 +- src/test/server/Server_test.cpp | 3 +- src/test/unit_test/SuiteJournal.h | 10 +- src/tests/libxrpl/basics/log.cpp | 72 +++-- src/xrpld/app/main/Application.cpp | 15 +- src/xrpld/app/main/Application.h | 6 +- 12 files changed, 294 insertions(+), 337 deletions(-) diff --git a/include/xrpl/basics/Log.h b/include/xrpl/basics/Log.h index fe61f2c5413..95d4150e2f1 100644 --- a/include/xrpl/basics/Log.h +++ b/include/xrpl/basics/Log.h @@ -68,8 +68,7 @@ class Logs operator=(Sink const&) = delete; void - write(beast::severities::Severity level, std::string&& text) - override; + write(beast::severities::Severity level, std::string&& text) override; void writeAlways(beast::severities::Severity level, std::string&& text) @@ -187,11 +186,19 @@ class Logs beast::Journal::Sink& operator[](std::string const& name); + template beast::Journal - journal( - std::string const& name, - std::optional attributes = - std::nullopt); + journal(std::string const& name, AttributesFactory&& factory) + { + return beast::Journal{ + get(name), name, std::forward(factory)}; + } + + beast::Journal + journal(std::string const& name) + { + return beast::Journal{get(name), name}; + } beast::severities::Severity threshold() const; diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index c3e0012aef9..54974ec5368 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -22,15 +22,6 @@ #include -#include -#include -#include - -#include -#include -#include - -#include #include #include #include @@ -86,6 +77,99 @@ operator<<(std::ostream& os, LogParameter const& param); namespace beast { +class SimpleJsonWriter +{ +public: + explicit SimpleJsonWriter(std::ostringstream& stream) : stream_(stream) + { + } + + void + startObject() const + { + stream_ << "{"; + } + void + endObject() const + { + stream_.seekp(-1, std::ios_base::end); + stream_ << "},"; + } + void + writeKey(std::string_view key) const + { + writeString(key); + stream_.seekp(-1, std::ios_base::end); + stream_ << ":"; + } + void + startArray() const + { + stream_ << "["; + } + void + endArray() const + { + stream_.seekp(-1, std::ios_base::end); + stream_ << "],"; + } + void + writeString(std::string_view str) const + { + stream_ << "\""; + escape(str, stream_); + stream_ << "\","; + } + void + writeInt(std::int64_t val) const + { + stream_ << val << ","; + } + void + writeUInt(std::uint64_t val) const + { + stream_ << val << ","; + } + void + writeDouble(double val) const + { + stream_ << val << ","; + } + void + writeBool(bool val) const + { + stream_ << (val ? "true," : "false,"); + } + void + writeNull() const + { + stream_ << "null" << ","; + } + void + writeRaw(std::string_view str) const + { + stream_ << str; + } + + [[nodiscard]] std::string + str() const + { + auto result = stream_.str(); + result.pop_back(); + return result; + } + +private: + static void + escape(std::string_view str, std::ostringstream& os) + { + // TODO: Support it + os << str; + } + + std::ostringstream& stream_; +}; + /** A namespace for easy access to logging severity values. */ namespace severities { /** Severity level / threshold of a Journal message. */ @@ -136,94 +220,36 @@ class Journal class Sink; - class JsonLogAttributes - { - public: - using AttributeFields = rapidjson::Value; - - JsonLogAttributes(); - JsonLogAttributes(JsonLogAttributes const& other); - - JsonLogAttributes& - operator=(JsonLogAttributes const& other); - - void - setModuleName(std::string const& name); - - void - combine(AttributeFields const& from); - - AttributeFields& - contextValues() - { - return contextValues_; - } - - [[nodiscard]] AttributeFields const& - contextValues() const - { - return contextValues_; - } - - rapidjson::MemoryPoolAllocator<>& - allocator() - { - return allocator_; - } - - private: - AttributeFields contextValues_; - rapidjson::MemoryPoolAllocator<> allocator_; - - friend class Journal; - }; - class JsonLogContext { - rapidjson::StringBuffer buffer_; - rapidjson::Writer messageParamsWriter_; - std::optional journalAttributesJson_; + std::ostringstream buffer_; + SimpleJsonWriter messageParamsWriter_; public: - JsonLogContext() - : messageParamsWriter_(buffer_) + JsonLogContext() : messageParamsWriter_(buffer_) { - } - rapidjson::Writer& + SimpleJsonWriter& writer() { return messageParamsWriter_; } - char const* - messageParams() - { - return buffer_.GetString(); - } - - std::optional& - journalAttributesJson() - { - return journalAttributesJson_; - } - void reset( std::source_location location, severities::Severity severity, - std::optional const& journalAttributesJson) noexcept; + std::string const& journalAttributesJson) noexcept; }; private: // Severity level / threshold of a Journal message. using Severity = severities::Severity; - std::optional m_attributes; - std::optional m_attributesJson; - static std::optional globalLogAttributes_; - static std::optional globalLogAttributesJson_; + std::string m_name; + std::string m_attributesJson; + static std::string globalLogAttributesJson_; static std::mutex globalLogAttributesMutex_; static bool m_jsonLogsEnabled; @@ -240,9 +266,6 @@ class Journal static std::string formatLog(std::string&& message); - void - rebuildAttributeJson(); - public: //-------------------------------------------------------------------------- @@ -471,42 +494,46 @@ class Journal /** Journal has no default constructor. */ Journal() = delete; - template - Journal( - Journal const& other, - TAttributesFactory&& attributesFactory = nullptr) - : m_attributes(other.m_attributes) - , m_sink(other.m_sink) + Journal(Journal const& other) + : m_name(other.m_name) , m_attributesJson(other.m_attributesJson) + , m_sink(other.m_sink) { -/* - if constexpr (!std::is_same_v, std::nullptr_t>) + } + + template + Journal(Journal const& other, TAttributesFactory&& attributesFactory) + : m_name(other.m_name), m_sink(other.m_sink) + { + std::ostringstream stream{other.m_attributesJson, std::ios_base::app}; + SimpleJsonWriter writer{stream}; + if (other.m_attributesJson.empty()) { - if (attributes.has_value()) - { - if (m_attributes) - m_attributes->combine(attributes->contextValues_); - else - m_attributes = std::move(attributes); - } - rebuildAttributeJson(); + writer.startObject(); } -*/ + attributesFactory(writer); + m_attributesJson = stream.str(); + } + + /** Create a journal that writes to the specified sink. */ + explicit Journal(Sink& sink, std::string const& name = {}) + : m_name(name), m_sink(&sink) + { } /** Create a journal that writes to the specified sink. */ + template explicit Journal( Sink& sink, - std::string const& name = {}, - std::optional attributes = std::nullopt) - : m_sink(&sink) - { - if (attributes) - { - m_attributes = std::move(attributes); - m_attributes->setModuleName(name); - } - rebuildAttributeJson(); + std::string const& name, + TAttributesFactory&& attributesFactory) + : m_name(name), m_sink(&sink) + { + std::ostringstream stream; + SimpleJsonWriter writer{stream}; + writer.startObject(); + attributesFactory(writer); + m_attributesJson = stream.str(); } Journal& @@ -516,8 +543,8 @@ class Journal return *this; m_sink = other.m_sink; - m_attributes = other.m_attributes; - rebuildAttributeJson(); + m_name = other.m_name; + m_attributesJson = other.m_attributesJson; return *this; } @@ -525,8 +552,8 @@ class Journal operator=(Journal&& other) noexcept { m_sink = other.m_sink; - m_attributes = std::move(other.m_attributes); - rebuildAttributeJson(); + m_name = std::move(other.m_name); + m_attributesJson = std::move(other.m_attributesJson); return *this; } @@ -539,7 +566,9 @@ class Journal /** Returns a stream for this sink, with the specified severity level. */ Stream - stream(Severity level, std::source_location location = std::source_location::current()) const + stream( + Severity level, + std::source_location location = std::source_location::current()) const { if (m_jsonLogsEnabled) initMessageContext(location, level); @@ -585,8 +614,6 @@ class Journal Stream warn(std::source_location location = std::source_location::current()) const { - char const* a = "a"; - rapidjson::Value v{a, 1}; if (m_jsonLogsEnabled) initMessageContext(location, severities::kWarning); return {*m_sink, severities::kWarning}; @@ -613,12 +640,26 @@ class Journal resetGlobalAttributes() { std::lock_guard lock(globalLogAttributesMutex_); - globalLogAttributes_ = std::nullopt; - globalLogAttributesJson_ = std::nullopt; + globalLogAttributesJson_.clear(); } + template static void - addGlobalAttributes(JsonLogAttributes globalLogAttributes); + addGlobalAttributes(TAttributesFactory&& factory) + { + std::lock_guard lock(globalLogAttributesMutex_); + + auto isEmpty = globalLogAttributesJson_.empty(); + std::ostringstream stream{ + std::move(globalLogAttributesJson_), std::ios_base::app}; + SimpleJsonWriter writer{stream}; + if (isEmpty) + { + writer.startObject(); + } + factory(writer); + globalLogAttributesJson_ = stream.str(); + } }; #ifndef __INTELLISENSE__ @@ -728,25 +769,25 @@ using logwstream = basic_logstream; namespace ripple::log { namespace detail { -template +template void setJsonValue( - rapidjson::Writer& writer, + beast::SimpleJsonWriter& writer, char const* name, T&& value, std::ostream* outStream) { using ValueType = std::decay_t; - writer.Key(name); + writer.writeKey(name); if constexpr (std::is_integral_v) { if constexpr (std::is_signed_v) { - writer.Int64(value); + writer.writeInt(value); } else { - writer.Uint64(value); + writer.writeUInt(value); } if (outStream) { @@ -755,7 +796,7 @@ setJsonValue( } else if constexpr (std::is_floating_point_v) { - writer.Double(value); + writer.writeDouble(value); if (outStream) { @@ -764,15 +805,17 @@ setJsonValue( } else if constexpr (std::is_same_v) { - writer.Bool(value); + writer.writeBool(value); if (outStream) { (*outStream) << value; } } - else if constexpr (std::is_same_v || std::is_same_v) + else if constexpr ( + std::is_same_v || + std::is_same_v) { - writer.String(value); + writer.writeString(value); if (outStream) { (*outStream) << value; @@ -780,7 +823,7 @@ setJsonValue( } else if constexpr (std::is_same_v) { - writer.String(value.c_str(), value.length()); + writer.writeString(value); if (outStream) { (*outStream) << value; @@ -791,11 +834,13 @@ setJsonValue( std::ostringstream oss; oss << value; - writer.String(oss.str().c_str(), oss.str().length()); + auto str = oss.str(); + + writer.writeString(str); if (outStream) { - (*outStream) << oss.str(); + (*outStream) << str; } } } @@ -847,13 +892,8 @@ template [[nodiscard]] auto attributes(Pair&&... pairs) { - return [&](rapidjson::Writer>& writer) { - (detail::setJsonValue( - writer, - pairs.first, - pairs.second, - nullptr), - ...); + return [&](beast::SimpleJsonWriter& writer) { + (detail::setJsonValue(writer, pairs.first, pairs.second, nullptr), ...); }; } diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index 60d4fadb8bb..197b4e16e33 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -56,9 +56,7 @@ Logs::Sink::write(beast::severities::Severity level, std::string&& text) } void -Logs::Sink::writeAlways( - beast::severities::Severity level, - std::string&& text) +Logs::Sink::writeAlways(beast::severities::Severity level, std::string&& text) { logs_.write(level, partition_, std::move(text), console()); } @@ -162,14 +160,6 @@ Logs::operator[](std::string const& name) return get(name); } -beast::Journal -Logs::journal( - std::string const& name, - std::optional attributes) -{ - return beast::Journal{get(name), name, std::move(attributes)}; -} - beast::severities::Severity Logs::threshold() const { diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index b900cd5e409..443bb683e06 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -19,10 +19,6 @@ #include -#include -#include -#include - #include #include #include @@ -31,9 +27,7 @@ namespace beast { -std::optional Journal::globalLogAttributes_; -std::optional Journal::globalLogAttributesJson_; - +std::string Journal::globalLogAttributesJson_; std::mutex Journal::globalLogAttributesMutex_; bool Journal::m_jsonLogsEnabled = false; thread_local Journal::JsonLogContext Journal::currentJsonLogContext_{}; @@ -125,75 +119,11 @@ severities::to_string(Severity severity) return ""; } -Journal::JsonLogAttributes::JsonLogAttributes() -{ - contextValues_.SetObject(); -} - -Journal::JsonLogAttributes::JsonLogAttributes(JsonLogAttributes const& other) -{ - contextValues_.SetObject(); - contextValues_.CopyFrom(other.contextValues_, allocator_); -} - -Journal::JsonLogAttributes& -Journal::JsonLogAttributes::operator=(JsonLogAttributes const& other) -{ - if (&other == this) - { - return *this; - } - contextValues_.CopyFrom(other.contextValues_, allocator_); - return *this; -} - -void -Journal::JsonLogAttributes::setModuleName(std::string const& name) -{ - contextValues_.AddMember( - rapidjson::StringRef("Module"), - rapidjson::Value{name.c_str(), allocator_}, - allocator_); -} - -void -Journal::JsonLogAttributes::combine( - AttributeFields const& from) -{ - for (auto& member : from.GetObject()) - { - contextValues_.RemoveMember(member.name); - - contextValues_.AddMember( - rapidjson::Value{member.name, allocator_}, - rapidjson::Value{member.value, allocator_}, - allocator_); - } -} - -void -Journal::addGlobalAttributes(JsonLogAttributes globalLogAttributes) -{ - std::lock_guard lock(globalLogAttributesMutex_); - if (!globalLogAttributes_) - { - globalLogAttributes_ = JsonLogAttributes{}; - } - globalLogAttributes_->combine(globalLogAttributes.contextValues()); - - rapidjson::StringBuffer buffer; - rapidjson::Writer writer{buffer}; - - globalLogAttributes_->contextValues().Accept(writer); - - globalLogAttributesJson_ = {buffer.GetString()}; -} - void Journal::JsonLogContext::reset( std::source_location location, severities::Severity severity, - std::optional const& journalAttributesJson) noexcept + std::string const& journalAttributesJson) noexcept { struct ThreadIdStringInitializer { @@ -207,32 +137,47 @@ Journal::JsonLogContext::reset( }; thread_local ThreadIdStringInitializer const threadId; - journalAttributesJson_ = journalAttributesJson; + buffer_.str(""); + + writer().startObject(); + + if (!journalAttributesJson.empty()) + { + writer().writeKey("JournalParams"); + writer().writeRaw(journalAttributesJson); + writer().endObject(); + } - buffer_.Clear(); + if (!globalLogAttributesJson_.empty()) + { + writer().writeKey("GlobalParams"); + writer().writeRaw(globalLogAttributesJson_); + writer().endObject(); + } - writer().StartObject(); + writer().writeKey("MessageParams"); + writer().startObject(); - writer().Key("Function"); - writer().String(location.function_name()); + writer().writeKey("Function"); + writer().writeString(location.function_name()); - writer().Key("File"); - writer().String(location.file_name()); + writer().writeKey("File"); + writer().writeString(location.file_name()); - writer().Key("Line"); - writer().Int64(location.line()); + writer().writeKey("Line"); + writer().writeInt(location.line()); - writer().Key("ThreadId"); - writer().String(threadId.value.c_str()); + writer().writeKey("ThreadId"); + writer().writeString(threadId.value); auto severityStr = to_string(severity); - writer().Key("Level"); - writer().String(severityStr.c_str()); + writer().writeKey("Level"); + writer().writeString(severityStr); - writer().Key("Time"); - writer().Uint64(std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count()); + writer().writeKey("Time"); + writer().writeUInt(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count()); } void @@ -251,58 +196,16 @@ Journal::formatLog(std::string&& message) return message; } - currentJsonLogContext_.writer().EndObject(); - auto messageParams = currentJsonLogContext_.messageParams(); - - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - - writer.StartObject(); - if (globalLogAttributesJson_.has_value()) - { - writer.Key("GlobalParams"); - writer.RawValue( - globalLogAttributesJson_->c_str(), - globalLogAttributesJson_->length(), - rapidjson::kObjectType); - } - - if (currentJsonLogContext_.journalAttributesJson().has_value()) - { - writer.Key("JournalParams"); - writer.RawValue( - currentJsonLogContext_.journalAttributesJson()->c_str(), - currentJsonLogContext_.journalAttributesJson()->length(), - rapidjson::kObjectType); - } - - writer.Key("MessageParams"); - writer.RawValue(messageParams, std::strlen(messageParams), rapidjson::kObjectType); - - writer.Key("Message"); - writer.String(message.c_str(), message.length()); - - writer.EndObject(); + auto& writer = currentJsonLogContext_.writer(); - return {buffer.GetString()}; -} + writer.endObject(); -void -Journal::rebuildAttributeJson() -{ - if (m_attributes.has_value()) - { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer{buffer}; + writer.writeKey("Message"); + writer.writeString(message); - m_attributes->contextValues().Accept(writer); + writer.endObject(); - m_attributesJson = {buffer.GetString()}; - } - else - { - m_attributesJson = {}; - } + return writer.str(); } void diff --git a/src/test/csf/Sim.h b/src/test/csf/Sim.h index cf4725c7098..b61cbbeb1ac 100644 --- a/src/test/csf/Sim.h +++ b/src/test/csf/Sim.h @@ -59,8 +59,7 @@ class BasicSink : public beast::Journal::Sink } void - writeAlways(beast::severities::Severity level, std::string&& text) - override + writeAlways(beast::severities::Severity level, std::string&& text) override { std::cout << clock_.now().time_since_epoch().count() << " " << text << std::endl; diff --git a/src/test/jtx/CaptureLogs.h b/src/test/jtx/CaptureLogs.h index 20bf4dd51aa..c89f1f74fac 100644 --- a/src/test/jtx/CaptureLogs.h +++ b/src/test/jtx/CaptureLogs.h @@ -57,8 +57,7 @@ class CaptureLogs : public Logs } void - write(beast::severities::Severity level, std::string&& text) - override + write(beast::severities::Severity level, std::string&& text) override { std::lock_guard lock(strmMutex_); strm_ << text; diff --git a/src/test/jtx/CheckMessageLogs.h b/src/test/jtx/CheckMessageLogs.h index 12673a8a142..e05ac5e4cfb 100644 --- a/src/test/jtx/CheckMessageLogs.h +++ b/src/test/jtx/CheckMessageLogs.h @@ -45,8 +45,7 @@ class CheckMessageLogs : public Logs } void - write(beast::severities::Severity level, std::string&& text) - override + write(beast::severities::Severity level, std::string&& text) override { if (text.find(owner_.msg_) != std::string::npos) *owner_.pFound_ = true; diff --git a/src/test/server/Server_test.cpp b/src/test/server/Server_test.cpp index 08b18312573..db1c58c2bc1 100644 --- a/src/test/server/Server_test.cpp +++ b/src/test/server/Server_test.cpp @@ -92,8 +92,7 @@ class Server_test : public beast::unit_test::suite } void - write(beast::severities::Severity level, std::string&& text) - override + write(beast::severities::Severity level, std::string&& text) override { if (level < threshold()) return; diff --git a/src/test/unit_test/SuiteJournal.h b/src/test/unit_test/SuiteJournal.h index f0421080b6a..8fb0f122bdc 100644 --- a/src/test/unit_test/SuiteJournal.h +++ b/src/test/unit_test/SuiteJournal.h @@ -52,14 +52,11 @@ class SuiteJournalSink : public beast::Journal::Sink write(beast::severities::Severity level, std::string&& text) override; void - writeAlways(beast::severities::Severity level, std::string&& text) - override; + writeAlways(beast::severities::Severity level, std::string&& text) override; }; inline void -SuiteJournalSink::write( - beast::severities::Severity level, - std::string&& text) +SuiteJournalSink::write(beast::severities::Severity level, std::string&& text) { // Only write the string if the level at least equals the threshold. if (level >= threshold()) @@ -145,8 +142,7 @@ class StreamSink : public beast::Journal::Sink } inline void - writeAlways(beast::severities::Severity level, std::string&& text) - override + writeAlways(beast::severities::Severity level, std::string&& text) override { strm_ << text << std::endl; } diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index 4c2b14eb988..6ab1eec5761 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -49,8 +49,7 @@ class MockLogs : public Logs operator=(Sink const&) = delete; void - write(beast::severities::Severity level, std::string&& text) - override + write(beast::severities::Severity level, std::string&& text) override { logs_.logStream_ << text; } @@ -168,7 +167,8 @@ TEST_CASE("Global attributes") CHECK(jsonLog["GlobalParams"].IsObject()); CHECK(jsonLog["GlobalParams"].HasMember("Field1")); CHECK(jsonLog["GlobalParams"]["Field1"].IsString()); - CHECK(jsonLog["GlobalParams"]["Field1"].GetString() == std::string{"Value1"}); + CHECK( + jsonLog["GlobalParams"]["Field1"].GetString() == std::string{"Value1"}); beast::Journal::disableStructuredJournal(); } @@ -198,9 +198,14 @@ TEST_CASE("Global attributes inheritable") CHECK(jsonLog.IsObject()); CHECK(jsonLog["GlobalParams"].HasMember("Field1")); CHECK(jsonLog["GlobalParams"]["Field1"].IsString()); - CHECK(jsonLog["GlobalParams"]["Field1"].GetString() == std::string{"Value1"}); - CHECK(jsonLog["JournalParams"]["Field1"].GetString() == std::string{"Value3"}); - CHECK(jsonLog["JournalParams"]["Field2"].GetString() == std::string{"Value2"}); + CHECK( + jsonLog["GlobalParams"]["Field1"].GetString() == std::string{"Value1"}); + CHECK( + jsonLog["JournalParams"]["Field1"].GetString() == + std::string{"Value3"}); + CHECK( + jsonLog["JournalParams"]["Field2"].GetString() == + std::string{"Value2"}); beast::Journal::disableStructuredJournal(); } @@ -224,8 +229,7 @@ class MockSink : public beast::Journal::Sink } void - writeAlways(beast::severities::Severity level, std::string&& text) - override + writeAlways(beast::severities::Severity level, std::string&& text) override { strm_ << text; } @@ -235,9 +239,8 @@ class JsonLogStreamFixture { public: JsonLogStreamFixture() - : sink_(beast::severities::kAll, logStream_), j_(sink_, "Test", log::attributes( - log::attr("Field1", "Value1") - )) + : sink_(beast::severities::kAll, logStream_) + , j_(sink_, "Test", log::attributes(log::attr("Field1", "Value1"))) { beast::Journal::enableStructuredJournal(); } @@ -267,9 +270,11 @@ class JsonLogStreamFixture TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") { - beast::Journal::addGlobalAttributes(log::attributes(log::attr("Field2", "Value2"))); + beast::Journal::addGlobalAttributes( + log::attributes(log::attr("Field2", "Value2"))); journal().debug() << std::boolalpha << true << std::noboolalpha << " Test " - << std::boolalpha << false << log::field("Field3", "Value3"); + << std::boolalpha << false + << log::field("Field3", "Value3"); rapidjson::Document logValue; logValue.Parse(stream().str().c_str()); @@ -484,7 +489,9 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributes") logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); CHECK(logValue["JournalParams"]["Field1"].IsString()); - CHECK(logValue["JournalParams"]["Field1"].GetString() == std::string{"Value1"}); + CHECK( + logValue["JournalParams"]["Field1"].GetString() == + std::string{"Value1"}); CHECK(logValue["JournalParams"]["Field2"].IsNumber()); CHECK(logValue["JournalParams"]["Field2"].GetInt() == 2); } @@ -494,9 +501,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributesInheritable") beast::Journal j{ journal(), log::attributes(log::attr("Field1", "Value1"), log::attr("Field2", 2))}; - beast::Journal j2{ - j, - log::attributes(log::attr("Field3", "Value3"), log::attr("Field2", 0))}; + beast::Journal j2{j, log::attributes(log::attr("Field3", "Value3"))}; j2.debug() << "Test"; @@ -507,12 +512,15 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributesInheritable") logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); CHECK(logValue["JournalParams"]["Field1"].IsString()); - CHECK(logValue["JournalParams"]["Field1"].GetString() == std::string{"Value1"}); + CHECK( + logValue["JournalParams"]["Field1"].GetString() == + std::string{"Value1"}); CHECK(logValue["JournalParams"]["Field3"].IsString()); - CHECK(logValue["JournalParams"]["Field3"].GetString() == std::string{"Value3"}); - // Field2 should be overwritten to 0 + CHECK( + logValue["JournalParams"]["Field3"].GetString() == + std::string{"Value3"}); CHECK(logValue["JournalParams"]["Field2"].IsNumber()); - CHECK(logValue["JournalParams"]["Field2"].GetInt() == 0); + CHECK(logValue["JournalParams"]["Field2"].GetInt() == 2); } TEST_CASE_FIXTURE( @@ -522,9 +530,7 @@ TEST_CASE_FIXTURE( beast::Journal j{ journal(), log::attributes(log::attr("Field1", "Value1"), log::attr("Field2", 2))}; - beast::Journal j2{ - j, - log::attributes(log::attr("Field3", "Value3"), log::attr("Field2", 0))}; + beast::Journal j2{j, log::attributes(log::attr("Field3", "Value3"))}; j2.debug() << "Test"; @@ -535,12 +541,16 @@ TEST_CASE_FIXTURE( logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); CHECK(logValue["JournalParams"]["Field1"].IsString()); - CHECK(logValue["JournalParams"]["Field1"].GetString() == std::string{"Value1"}); + CHECK( + logValue["JournalParams"]["Field1"].GetString() == + std::string{"Value1"}); CHECK(logValue["JournalParams"]["Field3"].IsString()); - CHECK(logValue["JournalParams"]["Field3"].GetString() == std::string{"Value3"}); + CHECK( + logValue["JournalParams"]["Field3"].GetString() == + std::string{"Value3"}); // Field2 should be overwritten to 0 CHECK(logValue["JournalParams"]["Field2"].IsNumber()); - CHECK(logValue["JournalParams"]["Field2"].GetInt() == 0); + CHECK(logValue["JournalParams"]["Field2"].GetInt() == 2); } TEST_CASE_FIXTURE( @@ -564,7 +574,9 @@ TEST_CASE_FIXTURE( logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); CHECK(logValue["JournalParams"]["Field1"].IsString()); - CHECK(logValue["JournalParams"]["Field1"].GetString() == std::string{"Value1"}); + CHECK( + logValue["JournalParams"]["Field1"].GetString() == + std::string{"Value1"}); CHECK(logValue["JournalParams"]["Field2"].IsNumber()); CHECK(logValue["JournalParams"]["Field2"].GetInt() == 2); } @@ -590,7 +602,9 @@ TEST_CASE_FIXTURE( logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); CHECK(logValue["JournalParams"]["Field1"].IsString()); - CHECK(logValue["JournalParams"]["Field1"].GetString() == std::string{"Value1"}); + CHECK( + logValue["JournalParams"]["Field1"].GetString() == + std::string{"Value1"}); CHECK(logValue["JournalParams"]["Field2"].IsNumber()); CHECK(logValue["JournalParams"]["Field2"].GetInt() == 2); } \ No newline at end of file diff --git a/src/xrpld/app/main/Application.cpp b/src/xrpld/app/main/Application.cpp index 43b389c2703..98f48f66578 100644 --- a/src/xrpld/app/main/Application.cpp +++ b/src/xrpld/app/main/Application.cpp @@ -834,8 +834,11 @@ class ApplicationImp : public Application, public BasicApp beast::Journal journal( std::string const& name, - std::optional attributes = - std::nullopt) override; + std::function const& attributes) + override; + + beast::Journal + journal(std::string const& name) override; //-------------------------------------------------------------------------- @@ -2175,11 +2178,17 @@ ApplicationImp::serverOkay(std::string& reason) beast::Journal ApplicationImp::journal( std::string const& name, - std::optional attributes) + std::function const& attributes) { return logs_->journal(name, std::move(attributes)); } +beast::Journal +ApplicationImp::journal(std::string const& name) +{ + return logs_->journal(name); +} + void ApplicationImp::setMaxDisallowedLedger() { diff --git a/src/xrpld/app/main/Application.h b/src/xrpld/app/main/Application.h index 6b8f5429421..d2127285927 100644 --- a/src/xrpld/app/main/Application.h +++ b/src/xrpld/app/main/Application.h @@ -260,8 +260,10 @@ class Application : public beast::PropertyStream::Source virtual beast::Journal journal( std::string const& name, - std::optional attributes = - std::nullopt) = 0; + std::function const& attributes) = 0; + + virtual beast::Journal + journal(std::string const& name) = 0; /* Returns the number of file descriptors the application needs */ virtual int From 7debf3e9f4329f5c71f63bbfd3d8d7e5a03d8650 Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 2 Sep 2025 00:07:17 +0100 Subject: [PATCH 33/88] Optimisation Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 146 ++++++++++++++++---- src/libxrpl/beast/utility/beast_Journal.cpp | 1 + src/tests/libxrpl/basics/log.cpp | 35 +++++ 3 files changed, 153 insertions(+), 29 deletions(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 54974ec5368..c401cc75487 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -82,73 +82,77 @@ class SimpleJsonWriter public: explicit SimpleJsonWriter(std::ostringstream& stream) : stream_(stream) { + stream_.imbue(std::locale::classic()); } void startObject() const { - stream_ << "{"; + stream_.put('{'); } void endObject() const { stream_.seekp(-1, std::ios_base::end); - stream_ << "},"; + stream_.write("},", 2); } void writeKey(std::string_view key) const { writeString(key); stream_.seekp(-1, std::ios_base::end); - stream_ << ":"; + stream_.put(':'); } void startArray() const { - stream_ << "["; + stream_.put('['); } void endArray() const { stream_.seekp(-1, std::ios_base::end); - stream_ << "],"; + stream_.write("],", 2); } void writeString(std::string_view str) const { - stream_ << "\""; + stream_.put('"'); escape(str, stream_); - stream_ << "\","; + stream_.write("\",", 2); } - void + std::string_view writeInt(std::int64_t val) const { - stream_ << val << ","; + return pushNumber(val, stream_); } - void + std::string_view writeUInt(std::uint64_t val) const { - stream_ << val << ","; + return pushNumber(val, stream_); } - void + std::string_view writeDouble(double val) const { - stream_ << val << ","; + return pushNumber(val, stream_); } - void + std::string_view writeBool(bool val) const { - stream_ << (val ? "true," : "false,"); + auto str = val ? "true," : "false,"; + stream_.write(str, std::strlen(str)); + return str; } void writeNull() const { - stream_ << "null" << ","; + stream_.write("null", std::strlen("null")); + stream_.put(','); } void writeRaw(std::string_view str) const { - stream_ << str; + stream_.write(str.data(), str.length()); } [[nodiscard]] std::string @@ -160,11 +164,69 @@ class SimpleJsonWriter } private: + template + static std::string_view + pushNumber(T val, std::ostringstream& stream) + { + static char buffer[128]; + auto result = std::to_chars(std::begin(buffer), std::end(buffer), val); + *result.ptr = ','; + auto len = result.ptr - std::begin(buffer); + stream.write(buffer, len + 1); + return {buffer, static_cast(len)}; + } + static void escape(std::string_view str, std::ostringstream& os) { - // TODO: Support it - os << str; + static constexpr char HEX[] = "0123456789ABCDEF"; + + const char* p = str.data(); + const char* end = p + str.size(); + const char* chunk = p; + + while (p < end) { + auto c = static_cast(*p); + + // JSON requires escaping for <0x20 and the two specials below. + bool needsEscape = (c < 0x20) || (c == '"') || (c == '\\'); + + if (!needsEscape) { + ++p; + continue; + } + + // Flush the preceding safe run in one go. + if (chunk != p) + os.write(chunk, p - chunk); + + switch (c) { + case '"': os.write("\\\"", 2); break; + case '\\': os.write("\\\\", 2); break; + case '\b': os.write("\\b", 2); break; + case '\f': os.write("\\f", 2); break; + case '\n': os.write("\\n", 2); break; + case '\r': os.write("\\r", 2); break; + case '\t': os.write("\\t", 2); break; + default: { + // Other C0 controls -> \u00XX (JSON compliant) + char buf[6]{ + '\\','u','0','0', + HEX[(c >> 4) & 0xF], + HEX[c & 0xF] + }; + os.write(buf, 6); + break; + } + } + + ++p; + chunk = p; + } + + // Flush trailing safe run + if (chunk != p) + os.write(chunk, p - chunk); } std::ostringstream& stream_; @@ -769,6 +831,13 @@ using logwstream = basic_logstream; namespace ripple::log { namespace detail { + +template +concept CanToChars = requires (T val) +{ + { to_chars(std::declval(), std::declval(), val) } -> std::convertible_to; +}; + template void setJsonValue( @@ -781,34 +850,35 @@ setJsonValue( writer.writeKey(name); if constexpr (std::is_integral_v) { + std::string_view sv; if constexpr (std::is_signed_v) { - writer.writeInt(value); + sv = writer.writeInt(value); } else { - writer.writeUInt(value); + sv = writer.writeUInt(value); } if (outStream) { - (*outStream) << value; + outStream->write(sv.data(), sv.size()); } } else if constexpr (std::is_floating_point_v) { - writer.writeDouble(value); + auto sv = writer.writeDouble(value); if (outStream) { - (*outStream) << value; + outStream->write(sv.data(), sv.size()); } } else if constexpr (std::is_same_v) { - writer.writeBool(value); + auto sv = writer.writeBool(value); if (outStream) { - (*outStream) << value; + outStream->write(sv.data(), sv.size()); } } else if constexpr ( @@ -818,7 +888,7 @@ setJsonValue( writer.writeString(value); if (outStream) { - (*outStream) << value; + outStream->write(value, std::strlen(value)); } } else if constexpr (std::is_same_v) @@ -826,12 +896,30 @@ setJsonValue( writer.writeString(value); if (outStream) { - (*outStream) << value; + outStream->write(value.c_str(), value.length()); } } else { + if constexpr (CanToChars) + { + char buffer[1024]; + std::to_chars_result result = to_chars(std::begin(buffer), std::end(buffer), value); + if (result.ec == std::errc{}) + { + std::string_view sv; + sv = {std::begin(buffer), result.ptr}; + writer.writeString(sv); + if (outStream) + { + outStream->write(sv.data(), sv.size()); + } + return; + } + } + std::ostringstream oss; + oss.imbue(std::locale::classic()); oss << value; auto str = oss.str(); @@ -840,7 +928,7 @@ setJsonValue( if (outStream) { - (*outStream) << str; + outStream->write(str.c_str(), static_cast(str.size())); } } } diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index 443bb683e06..3c30ad2685c 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -131,6 +131,7 @@ Journal::JsonLogContext::reset( ThreadIdStringInitializer() { std::stringstream threadIdStream; + threadIdStream.imbue(std::locale::classic()); threadIdStream << std::this_thread::get_id(); value = threadIdStream.str(); } diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index 6ab1eec5761..6cbf9f908a9 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -209,6 +209,41 @@ TEST_CASE("Global attributes inheritable") beast::Journal::disableStructuredJournal(); } +TEST_CASE("Test JsonWriter") +{ + { + std::ostringstream stream; + beast::SimpleJsonWriter writer{stream}; + + writer.writeString("\n"); + CHECK(writer.str() == "\"\\n\""); + } + + { + std::ostringstream stream; + beast::SimpleJsonWriter writer{stream}; + + writer.writeString("\t"); + CHECK(writer.str() == "\"\\t\""); + } + + { + std::ostringstream stream; + beast::SimpleJsonWriter writer{stream}; + + writer.writeString(std::string_view{"\0", 1}); + CHECK(writer.str() == "\"\\u0000\""); + } + + { + std::ostringstream stream; + beast::SimpleJsonWriter writer{stream}; + + writer.writeString("\"\\"); + CHECK(writer.str() == "\"\\\"\\\\\""); + } +} + /** * @brief sink for writing all log messages to a stringstream */ From feae1d6e15abcd7dafae6afc9fffdeb0fb99160b Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 2 Sep 2025 00:28:28 +0100 Subject: [PATCH 34/88] Optimisation Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 80 ++++++++++----------- src/libxrpl/beast/utility/beast_Journal.cpp | 4 +- src/tests/libxrpl/basics/log.cpp | 16 ++--- 3 files changed, 48 insertions(+), 52 deletions(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index c401cc75487..4efa7e6b656 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -80,46 +80,44 @@ namespace beast { class SimpleJsonWriter { public: - explicit SimpleJsonWriter(std::ostringstream& stream) : stream_(stream) + explicit SimpleJsonWriter(std::string& stream) : stream_(stream) { - stream_.imbue(std::locale::classic()); } void startObject() const { - stream_.put('{'); + stream_.append("{", 1); } void endObject() const { - stream_.seekp(-1, std::ios_base::end); - stream_.write("},", 2); + stream_.pop_back(); + stream_.append("},", 2); } void writeKey(std::string_view key) const { writeString(key); - stream_.seekp(-1, std::ios_base::end); - stream_.put(':'); + *stream_.rbegin() = ':'; } void startArray() const { - stream_.put('['); + stream_.append("[", 1); } void endArray() const { - stream_.seekp(-1, std::ios_base::end); - stream_.write("],", 2); + stream_.pop_back(); + stream_.append("],", 2); } void writeString(std::string_view str) const { - stream_.put('"'); + stream_.push_back('"'); escape(str, stream_); - stream_.write("\",", 2); + stream_.append("\",", 2); } std::string_view writeInt(std::int64_t val) const @@ -140,44 +138,43 @@ class SimpleJsonWriter writeBool(bool val) const { auto str = val ? "true," : "false,"; - stream_.write(str, std::strlen(str)); + stream_.append(str, std::strlen(str)); return str; } void writeNull() const { - stream_.write("null", std::strlen("null")); - stream_.put(','); + stream_.append("null", std::strlen("null")); + stream_.push_back(','); } void writeRaw(std::string_view str) const { - stream_.write(str.data(), str.length()); + stream_.append(str); } [[nodiscard]] std::string - str() const + finish() { - auto result = stream_.str(); - result.pop_back(); - return result; + stream_.pop_back(); + return stream_; } private: template static std::string_view - pushNumber(T val, std::ostringstream& stream) + pushNumber(T val, std::string& stream) { static char buffer[128]; auto result = std::to_chars(std::begin(buffer), std::end(buffer), val); *result.ptr = ','; auto len = result.ptr - std::begin(buffer); - stream.write(buffer, len + 1); + stream.append(buffer, len + 1); return {buffer, static_cast(len)}; } static void - escape(std::string_view str, std::ostringstream& os) + escape(std::string_view str, std::string& os) { static constexpr char HEX[] = "0123456789ABCDEF"; @@ -198,16 +195,16 @@ class SimpleJsonWriter // Flush the preceding safe run in one go. if (chunk != p) - os.write(chunk, p - chunk); + os.append(chunk, p - chunk); switch (c) { - case '"': os.write("\\\"", 2); break; - case '\\': os.write("\\\\", 2); break; - case '\b': os.write("\\b", 2); break; - case '\f': os.write("\\f", 2); break; - case '\n': os.write("\\n", 2); break; - case '\r': os.write("\\r", 2); break; - case '\t': os.write("\\t", 2); break; + case '"': os.append("\\\"", 2); break; + case '\\': os.append("\\\\", 2); break; + case '\b': os.append("\\b", 2); break; + case '\f': os.append("\\f", 2); break; + case '\n': os.append("\\n", 2); break; + case '\r': os.append("\\r", 2); break; + case '\t': os.append("\\t", 2); break; default: { // Other C0 controls -> \u00XX (JSON compliant) char buf[6]{ @@ -215,7 +212,7 @@ class SimpleJsonWriter HEX[(c >> 4) & 0xF], HEX[c & 0xF] }; - os.write(buf, 6); + os.append(buf, 6); break; } } @@ -226,10 +223,10 @@ class SimpleJsonWriter // Flush trailing safe run if (chunk != p) - os.write(chunk, p - chunk); + os.append(chunk, p - chunk); } - std::ostringstream& stream_; + std::string& stream_; }; /** A namespace for easy access to logging severity values. */ @@ -284,7 +281,7 @@ class Journal class JsonLogContext { - std::ostringstream buffer_; + std::string buffer_; SimpleJsonWriter messageParamsWriter_; public: @@ -567,14 +564,14 @@ class Journal Journal(Journal const& other, TAttributesFactory&& attributesFactory) : m_name(other.m_name), m_sink(other.m_sink) { - std::ostringstream stream{other.m_attributesJson, std::ios_base::app}; + std::string stream{other.m_attributesJson}; SimpleJsonWriter writer{stream}; if (other.m_attributesJson.empty()) { writer.startObject(); } attributesFactory(writer); - m_attributesJson = stream.str(); + m_attributesJson = std::move(stream); } /** Create a journal that writes to the specified sink. */ @@ -591,11 +588,11 @@ class Journal TAttributesFactory&& attributesFactory) : m_name(name), m_sink(&sink) { - std::ostringstream stream; + std::string stream; SimpleJsonWriter writer{stream}; writer.startObject(); attributesFactory(writer); - m_attributesJson = stream.str(); + m_attributesJson = std::move(stream); } Journal& @@ -712,15 +709,14 @@ class Journal std::lock_guard lock(globalLogAttributesMutex_); auto isEmpty = globalLogAttributesJson_.empty(); - std::ostringstream stream{ - std::move(globalLogAttributesJson_), std::ios_base::app}; + std::string stream{std::move(globalLogAttributesJson_)}; SimpleJsonWriter writer{stream}; if (isEmpty) { writer.startObject(); } factory(writer); - globalLogAttributesJson_ = stream.str(); + globalLogAttributesJson_ = std::move(stream); } }; diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index 3c30ad2685c..8cc8a811f6f 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -138,7 +138,7 @@ Journal::JsonLogContext::reset( }; thread_local ThreadIdStringInitializer const threadId; - buffer_.str(""); + buffer_.clear(); writer().startObject(); @@ -206,7 +206,7 @@ Journal::formatLog(std::string&& message) writer.endObject(); - return writer.str(); + return writer.finish(); } void diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index 6cbf9f908a9..3f015072fc8 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -212,35 +212,35 @@ TEST_CASE("Global attributes inheritable") TEST_CASE("Test JsonWriter") { { - std::ostringstream stream; + std::string stream; beast::SimpleJsonWriter writer{stream}; writer.writeString("\n"); - CHECK(writer.str() == "\"\\n\""); + CHECK(writer.finish() == "\"\\n\""); } { - std::ostringstream stream; + std::string stream; beast::SimpleJsonWriter writer{stream}; writer.writeString("\t"); - CHECK(writer.str() == "\"\\t\""); + CHECK(writer.finish() == "\"\\t\""); } { - std::ostringstream stream; + std::string stream; beast::SimpleJsonWriter writer{stream}; writer.writeString(std::string_view{"\0", 1}); - CHECK(writer.str() == "\"\\u0000\""); + CHECK(writer.finish() == "\"\\u0000\""); } { - std::ostringstream stream; + std::string stream; beast::SimpleJsonWriter writer{stream}; writer.writeString("\"\\"); - CHECK(writer.str() == "\"\\\"\\\\\""); + CHECK(writer.finish() == "\"\\\"\\\\\""); } } From f50f76788b502cd71ba218f023975195a0deb47d Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 2 Sep 2025 00:32:35 +0100 Subject: [PATCH 35/88] Fix issues Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 4efa7e6b656..452306f2dad 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -22,7 +22,8 @@ #include -#include +#include +#include #include #include #include From 1d3d0c677417bcc324d96026a05b77341b89ce80 Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 2 Sep 2025 17:22:53 +0100 Subject: [PATCH 36/88] Optimise Signed-off-by: JCW --- cmake/RippledCore.cmake | 3 - conanfile.py | 3 +- include/xrpl/beast/utility/Journal.h | 115 +++--- src/libxrpl/beast/utility/beast_Journal.cpp | 26 +- src/tests/libxrpl/CMakeLists.txt | 3 +- src/tests/libxrpl/basics/log.cpp | 379 +++++++++----------- 6 files changed, 270 insertions(+), 259 deletions(-) diff --git a/cmake/RippledCore.cmake b/cmake/RippledCore.cmake index b9d5cb2cc60..2e41531ba7a 100644 --- a/cmake/RippledCore.cmake +++ b/cmake/RippledCore.cmake @@ -51,8 +51,6 @@ target_link_libraries(xrpl.libpb # TODO: Clean up the number of library targets later. add_library(xrpl.imports.main INTERFACE) -find_package(RapidJSON) - target_link_libraries(xrpl.imports.main INTERFACE LibArchive::LibArchive @@ -77,7 +75,6 @@ add_module(xrpl beast) target_link_libraries(xrpl.libxrpl.beast PUBLIC xrpl.imports.main xrpl.libpb - rapidjson ) # Level 02 diff --git a/conanfile.py b/conanfile.py index 024ef85c0dc..e0373ac65e5 100644 --- a/conanfile.py +++ b/conanfile.py @@ -29,8 +29,7 @@ class Xrpl(ConanFile): 'nudb/2.0.9', 'openssl/1.1.1w', 'soci/4.0.3', - 'zlib/1.3.1', - "rapidjson/1.1.0" + 'zlib/1.3.1' ] test_requires = [ diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 452306f2dad..5723006f093 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -23,9 +23,10 @@ #include #include -#include #include #include +#include +#include #include namespace ripple::log { @@ -88,37 +89,45 @@ class SimpleJsonWriter void startObject() const { - stream_.append("{", 1); + stream_.push_back('{'); } void endObject() const { + using namespace std::string_view_literals; stream_.pop_back(); - stream_.append("},", 2); + stream_.append("},"sv); } void writeKey(std::string_view key) const { writeString(key); - *stream_.rbegin() = ':'; + stream_.back() = ':'; } void startArray() const { - stream_.append("[", 1); + stream_.push_back('['); } void endArray() const { + using namespace std::string_view_literals; stream_.pop_back(); - stream_.append("],", 2); + stream_.append("],"sv); } void writeString(std::string_view str) const { + using namespace std::string_view_literals; stream_.push_back('"'); escape(str, stream_); - stream_.append("\",", 2); + stream_.append("\","sv); + } + std::string_view + writeInt(std::int32_t val) const + { + return pushNumber(val, stream_); } std::string_view writeInt(std::int64_t val) const @@ -126,6 +135,11 @@ class SimpleJsonWriter return pushNumber(val, stream_); } std::string_view + writeUInt(std::uint32_t val) const + { + return pushNumber(val, stream_); + } + std::string_view writeUInt(std::uint64_t val) const { return pushNumber(val, stream_); @@ -138,15 +152,16 @@ class SimpleJsonWriter std::string_view writeBool(bool val) const { - auto str = val ? "true," : "false,"; - stream_.append(str, std::strlen(str)); + using namespace std::string_view_literals; + auto str = val ? "true,"sv : "false,"sv; + stream_.append(str); return str; } void writeNull() const { - stream_.append("null", std::strlen("null")); - stream_.push_back(','); + using namespace std::string_view_literals; + stream_.append("null,"sv); } void writeRaw(std::string_view str) const @@ -154,11 +169,10 @@ class SimpleJsonWriter stream_.append(str); } - [[nodiscard]] std::string + [[nodiscard]] std::string_view finish() { - stream_.pop_back(); - return stream_; + return std::string_view{stream_.c_str(), stream_.size() - 1}; } private: @@ -166,10 +180,11 @@ class SimpleJsonWriter static std::string_view pushNumber(T val, std::string& stream) { - static char buffer[128]; + thread_local char buffer[128]; auto result = std::to_chars(std::begin(buffer), std::end(buffer), val); - *result.ptr = ','; - auto len = result.ptr - std::begin(buffer); + auto ptr = result.ptr; + *ptr = ','; + auto len = ptr - std::begin(buffer); stream.append(buffer, len + 1); return {buffer, static_cast(len)}; } @@ -179,17 +194,19 @@ class SimpleJsonWriter { static constexpr char HEX[] = "0123456789ABCDEF"; - const char* p = str.data(); - const char* end = p + str.size(); - const char* chunk = p; + char const* p = str.data(); + char const* end = p + str.size(); + char const* chunk = p; - while (p < end) { + while (p < end) + { auto c = static_cast(*p); // JSON requires escaping for <0x20 and the two specials below. bool needsEscape = (c < 0x20) || (c == '"') || (c == '\\'); - if (!needsEscape) { + if (!needsEscape) + { ++p; continue; } @@ -198,21 +215,33 @@ class SimpleJsonWriter if (chunk != p) os.append(chunk, p - chunk); - switch (c) { - case '"': os.append("\\\"", 2); break; - case '\\': os.append("\\\\", 2); break; - case '\b': os.append("\\b", 2); break; - case '\f': os.append("\\f", 2); break; - case '\n': os.append("\\n", 2); break; - case '\r': os.append("\\r", 2); break; - case '\t': os.append("\\t", 2); break; + switch (c) + { + case '"': + os.append("\\\"", 2); + break; + case '\\': + os.append("\\\\", 2); + break; + case '\b': + os.append("\\b", 2); + break; + case '\f': + os.append("\\f", 2); + break; + case '\n': + os.append("\\n", 2); + break; + case '\r': + os.append("\\r", 2); + break; + case '\t': + os.append("\\t", 2); + break; default: { // Other C0 controls -> \u00XX (JSON compliant) char buf[6]{ - '\\','u','0','0', - HEX[(c >> 4) & 0xF], - HEX[c & 0xF] - }; + '\\', 'u', '0', '0', HEX[(c >> 4) & 0xF], HEX[c & 0xF]}; os.append(buf, 6); break; } @@ -288,6 +317,7 @@ class Journal public: JsonLogContext() : messageParamsWriter_(buffer_) { + buffer_.reserve(1024 * 5); } SimpleJsonWriter& @@ -300,7 +330,7 @@ class Journal reset( std::source_location location, severities::Severity severity, - std::string const& journalAttributesJson) noexcept; + std::string_view journalAttributesJson) noexcept; }; private: @@ -323,7 +353,7 @@ class Journal std::source_location location, severities::Severity severity) const; - static std::string + static std::string_view formatLog(std::string&& message); public: @@ -830,9 +860,10 @@ namespace ripple::log { namespace detail { template -concept CanToChars = requires (T val) -{ - { to_chars(std::declval(), std::declval(), val) } -> std::convertible_to; +concept CanToChars = requires(T val) { + { + to_chars(std::declval(), std::declval(), val) + } -> std::convertible_to; }; template @@ -901,7 +932,8 @@ setJsonValue( if constexpr (CanToChars) { char buffer[1024]; - std::to_chars_result result = to_chars(std::begin(buffer), std::end(buffer), value); + std::to_chars_result result = + to_chars(std::begin(buffer), std::end(buffer), value); if (result.ec == std::errc{}) { std::string_view sv; @@ -925,7 +957,8 @@ setJsonValue( if (outStream) { - outStream->write(str.c_str(), static_cast(str.size())); + outStream->write( + str.c_str(), static_cast(str.size())); } } } diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index 8cc8a811f6f..74a42fb34a1 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -123,7 +123,7 @@ void Journal::JsonLogContext::reset( std::source_location location, severities::Severity severity, - std::string const& journalAttributesJson) noexcept + std::string_view journalAttributesJson) noexcept { struct ThreadIdStringInitializer { @@ -152,7 +152,9 @@ Journal::JsonLogContext::reset( if (!globalLogAttributesJson_.empty()) { writer().writeKey("GlobalParams"); - writer().writeRaw(globalLogAttributesJson_); + writer().writeRaw(std::string_view{ + std::begin(globalLogAttributesJson_), + std::end(globalLogAttributesJson_)}); writer().endObject(); } @@ -166,7 +168,7 @@ Journal::JsonLogContext::reset( writer().writeString(location.file_name()); writer().writeKey("Line"); - writer().writeInt(location.line()); + writer().writeInt(static_cast(location.line())); writer().writeKey("ThreadId"); writer().writeString(threadId.value); @@ -176,9 +178,9 @@ Journal::JsonLogContext::reset( writer().writeString(severityStr); writer().writeKey("Time"); - writer().writeUInt(std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count()); + writer().writeInt(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count()); } void @@ -186,10 +188,14 @@ Journal::initMessageContext( std::source_location location, severities::Severity severity) const { - currentJsonLogContext_.reset(location, severity, m_attributesJson); + currentJsonLogContext_.reset( + location, + severity, + std::string_view{ + std::begin(m_attributesJson), std::end(m_attributesJson)}); } -std::string +std::string_view Journal::formatLog(std::string&& message) { if (!m_jsonLogsEnabled) @@ -288,9 +294,9 @@ Journal::ScopedStream::~ScopedStream() if (!s.empty()) { if (s == "\n") - m_sink.write(m_level, formatLog("")); + m_sink.write(m_level, std::string{formatLog("")}); else - m_sink.write(m_level, formatLog(std::move(s))); + m_sink.write(m_level, std::string{formatLog(std::move(s))}); } } diff --git a/src/tests/libxrpl/CMakeLists.txt b/src/tests/libxrpl/CMakeLists.txt index 72f0e532ce8..73520a51282 100644 --- a/src/tests/libxrpl/CMakeLists.txt +++ b/src/tests/libxrpl/CMakeLists.txt @@ -2,11 +2,10 @@ include(xrpl_add_test) # Test requirements. find_package(doctest REQUIRED) -find_package(RapidJSON REQUIRED) # Common library dependencies for the rest of the tests. add_library(xrpl.imports.test INTERFACE) -target_link_libraries(xrpl.imports.test INTERFACE doctest::doctest rapidjson xrpl.libxrpl) +target_link_libraries(xrpl.imports.test INTERFACE doctest::doctest xrpl.libxrpl) # One test for each module. xrpl_add_test(basics) diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index 3f015072fc8..9bdb6f49ec6 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -20,7 +20,10 @@ #include #include -#include +#include + +#include +#include using namespace ripple; @@ -51,21 +54,21 @@ class MockLogs : public Logs void write(beast::severities::Severity level, std::string&& text) override { - logs_.logStream_ << text; + logs_.write(level, partition_, text, false); } void writeAlways(beast::severities::Severity level, std::string&& text) override { - logs_.logStream_ << text; + logs_.write(level, partition_, text, false); } }; - std::stringstream& logStream_; + std::string& logStream_; public: - MockLogs(std::stringstream& logStream, beast::severities::Severity level) + MockLogs(std::string& logStream, beast::severities::Severity level) : Logs(level), logStream_(logStream) { } @@ -77,23 +80,35 @@ class MockLogs : public Logs { return std::make_unique(partition, startingLevel, *this); } + + void + write( + beast::severities::Severity level, + std::string const& partition, + std::string const& text, + bool console) + { + std::string s; + format(s, text, level, partition); + logStream_.append(s); + } }; TEST_CASE("Text logs") { - std::stringstream logStream; + std::string logStream; MockLogs logs{logStream, beast::severities::kAll}; logs.journal("Test").debug() << "Test"; - CHECK(logStream.str().find("Test") != std::string::npos); + CHECK(logStream.find("Test") != std::string::npos); - logStream.str(""); + logStream.clear(); logs.journal("Test").debug() << "\n"; - CHECK(logStream.str().find("\n") == std::string::npos); + CHECK(logStream.find("\n") == std::string::npos); } TEST_CASE("Test format output") @@ -118,35 +133,34 @@ TEST_CASE("Test format output when structured logs are enabled") TEST_CASE("Enable json logs") { - std::stringstream logStream; + std::string logStream; MockLogs logs{logStream, beast::severities::kAll}; - logs.journal("Test").debug() << "Test"; + logs.journal("Test ").debug() << "Test123"; - CHECK(logStream.str() == "Test"); + CHECK(logStream.find("Test123") != std::string::npos); - logStream.str(""); + logStream.clear(); beast::Journal::enableStructuredJournal(); logs.journal("Test").debug() << "\n"; - rapidjson::Document doc; - doc.Parse(logStream.str().c_str()); - - CHECK(doc.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + boost::system::error_code ec; + auto doc = boost::json::parse(logStream, ec); + CHECK(ec == boost::system::errc::success); - CHECK(doc.IsObject()); - CHECK(doc.HasMember("Message")); - CHECK(doc["Message"].IsString()); - CHECK(doc["Message"].GetString() == std::string{""}); + CHECK(doc.is_object()); + CHECK(doc.as_object().contains("Message")); + CHECK(doc.as_object()["Message"].is_string()); + CHECK(doc.as_object()["Message"].get_string() == ""); beast::Journal::disableStructuredJournal(); } TEST_CASE("Global attributes") { - std::stringstream logStream; + std::string logStream; MockLogs logs{logStream, beast::severities::kAll}; @@ -156,25 +170,23 @@ TEST_CASE("Global attributes") logs.journal("Test").debug() << "Test"; - rapidjson::Document jsonLog; - jsonLog.Parse(logStream.str().c_str()); - - CHECK( - jsonLog.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + boost::system::error_code ec; + auto jsonLog = boost::json::parse(logStream, ec); + CHECK(ec == boost::system::errc::success); - CHECK(jsonLog.IsObject()); - CHECK(jsonLog.HasMember("GlobalParams")); - CHECK(jsonLog["GlobalParams"].IsObject()); - CHECK(jsonLog["GlobalParams"].HasMember("Field1")); - CHECK(jsonLog["GlobalParams"]["Field1"].IsString()); + CHECK(jsonLog.is_object()); + CHECK(jsonLog.as_object().contains("GlobalParams")); + CHECK(jsonLog.as_object()["GlobalParams"].is_object()); + CHECK(jsonLog.as_object()["GlobalParams"].as_object().contains("Field1")); + CHECK(jsonLog.as_object()["GlobalParams"].as_object()["Field1"].is_string()); CHECK( - jsonLog["GlobalParams"]["Field1"].GetString() == std::string{"Value1"}); + jsonLog.as_object()["GlobalParams"].as_object()["Field1"].get_string() == "Value1"); beast::Journal::disableStructuredJournal(); } TEST_CASE("Global attributes inheritable") { - std::stringstream logStream; + std::string logStream; MockLogs logs{logStream, beast::severities::kAll}; @@ -189,23 +201,19 @@ TEST_CASE("Global attributes inheritable") .debug() << "Test"; - rapidjson::Document jsonLog; - jsonLog.Parse(logStream.str().c_str()); - - CHECK( - jsonLog.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + boost::system::error_code ec; + auto jsonLog = boost::json::parse(logStream, ec); + CHECK(ec == boost::system::errc::success); - CHECK(jsonLog.IsObject()); - CHECK(jsonLog["GlobalParams"].HasMember("Field1")); - CHECK(jsonLog["GlobalParams"]["Field1"].IsString()); + CHECK(jsonLog.is_object()); + CHECK(jsonLog.as_object()["GlobalParams"].as_object().contains("Field1")); + CHECK(jsonLog.as_object()["GlobalParams"].as_object()["Field1"].is_string()); CHECK( - jsonLog["GlobalParams"]["Field1"].GetString() == std::string{"Value1"}); + jsonLog.as_object()["GlobalParams"].as_object()["Field1"].get_string() == "Value1"); CHECK( - jsonLog["JournalParams"]["Field1"].GetString() == - std::string{"Value3"}); + jsonLog.as_object()["JournalParams"].as_object()["Field1"].get_string() == "Value3"); CHECK( - jsonLog["JournalParams"]["Field2"].GetString() == - std::string{"Value2"}); + jsonLog.as_object()["JournalParams"].as_object()["Field2"].get_string() == "Value2"); beast::Journal::disableStructuredJournal(); } @@ -311,35 +319,33 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") << std::boolalpha << false << log::field("Field3", "Value3"); - rapidjson::Document logValue; - logValue.Parse(stream().str().c_str()); - - CHECK( - logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); - - CHECK(logValue.IsObject()); - CHECK(logValue.HasMember("GlobalParams")); - CHECK(logValue.HasMember("JournalParams")); - CHECK(logValue.HasMember("MessageParams")); - CHECK(logValue.HasMember("Message")); - - CHECK(logValue["GlobalParams"].IsObject()); - CHECK(logValue["JournalParams"].IsObject()); - CHECK(logValue["MessageParams"].IsObject()); - CHECK(logValue["Message"].IsString()); - - CHECK(logValue["MessageParams"].HasMember("Function")); - CHECK(logValue["MessageParams"].HasMember("File")); - CHECK(logValue["MessageParams"].HasMember("Line")); - CHECK(logValue["MessageParams"].HasMember("ThreadId")); - CHECK(logValue["MessageParams"].HasMember("Level")); - CHECK(logValue["MessageParams"].HasMember("Time")); - - CHECK(logValue["MessageParams"]["Function"].IsString()); - CHECK(logValue["MessageParams"]["File"].IsString()); - CHECK(logValue["MessageParams"]["Line"].IsNumber()); - - CHECK(logValue["Message"].GetString() == std::string{"true Test false"}); + boost::system::error_code ec; + auto logValue = boost::json::parse(stream().str(), ec); + CHECK(ec == boost::system::errc::success); + + CHECK(logValue.is_object()); + CHECK(logValue.as_object().contains("GlobalParams")); + CHECK(logValue.as_object().contains("JournalParams")); + CHECK(logValue.as_object().contains("MessageParams")); + CHECK(logValue.as_object().contains("Message")); + + CHECK(logValue.as_object()["GlobalParams"].is_object()); + CHECK(logValue.as_object()["JournalParams"].is_object()); + CHECK(logValue.as_object()["MessageParams"].is_object()); + CHECK(logValue.as_object()["Message"].is_string()); + + CHECK(logValue.as_object()["MessageParams"].as_object().contains("Function")); + CHECK(logValue.as_object()["MessageParams"].as_object().contains("File")); + CHECK(logValue.as_object()["MessageParams"].as_object().contains("Line")); + CHECK(logValue.as_object()["MessageParams"].as_object().contains("ThreadId")); + CHECK(logValue.as_object()["MessageParams"].as_object().contains("Level")); + CHECK(logValue.as_object()["MessageParams"].as_object().contains("Time")); + + CHECK(logValue.as_object()["MessageParams"].as_object()["Function"].is_string()); + CHECK(logValue.as_object()["MessageParams"].as_object()["File"].is_string()); + CHECK(logValue.as_object()["MessageParams"].as_object()["Line"].is_number()); + + CHECK(logValue.as_object()["Message"].get_string() == std::string{"true Test false"}); } TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") @@ -348,15 +354,12 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") stream().str(""); journal().trace() << "Test"; - rapidjson::Document logValue; - logValue.Parse(stream().str().c_str()); + boost::system::error_code ec; + auto logValue = boost::json::parse(stream().str(), ec); + CHECK(ec == boost::system::errc::success); CHECK( - logValue.GetParseError() == - rapidjson::ParseErrorCode::kParseErrorNone); - - CHECK( - logValue["MessageParams"]["Level"].GetString() == + logValue.as_object()["MessageParams"].as_object()["Level"].get_string() == beast::severities::to_string(beast::severities::kTrace)); } @@ -364,15 +367,12 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") stream().str(""); journal().debug() << "Test"; - rapidjson::Document logValue; - logValue.Parse(stream().str().c_str()); - - CHECK( - logValue.GetParseError() == - rapidjson::ParseErrorCode::kParseErrorNone); + boost::system::error_code ec; + auto logValue = boost::json::parse(stream().str(), ec); + CHECK(ec == boost::system::errc::success); CHECK( - logValue["MessageParams"]["Level"].GetString() == + logValue.as_object()["MessageParams"].as_object()["Level"].get_string() == beast::severities::to_string(beast::severities::kDebug)); } @@ -380,15 +380,12 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") stream().str(""); journal().info() << "Test"; - rapidjson::Document logValue; - logValue.Parse(stream().str().c_str()); - - CHECK( - logValue.GetParseError() == - rapidjson::ParseErrorCode::kParseErrorNone); + boost::system::error_code ec; + auto logValue = boost::json::parse(stream().str(), ec); + CHECK(ec == boost::system::errc::success); CHECK( - logValue["MessageParams"]["Level"].GetString() == + logValue.as_object()["MessageParams"].as_object()["Level"].get_string() == beast::severities::to_string(beast::severities::kInfo)); } @@ -396,15 +393,12 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") stream().str(""); journal().warn() << "Test"; - rapidjson::Document logValue; - logValue.Parse(stream().str().c_str()); + boost::system::error_code ec; + auto logValue = boost::json::parse(stream().str(), ec); + CHECK(ec == boost::system::errc::success); CHECK( - logValue.GetParseError() == - rapidjson::ParseErrorCode::kParseErrorNone); - - CHECK( - logValue["MessageParams"]["Level"].GetString() == + logValue.as_object()["MessageParams"].as_object()["Level"].get_string() == beast::severities::to_string(beast::severities::kWarning)); } @@ -412,15 +406,12 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") stream().str(""); journal().error() << "Test"; - rapidjson::Document logValue; - logValue.Parse(stream().str().c_str()); + boost::system::error_code ec; + auto logValue = boost::json::parse(stream().str(), ec); + CHECK(ec == boost::system::errc::success); CHECK( - logValue.GetParseError() == - rapidjson::ParseErrorCode::kParseErrorNone); - - CHECK( - logValue["MessageParams"]["Level"].GetString() == + logValue.as_object()["MessageParams"].as_object()["Level"].get_string() == beast::severities::to_string(beast::severities::kError)); } @@ -428,15 +419,12 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") stream().str(""); journal().fatal() << "Test"; - rapidjson::Document logValue; - logValue.Parse(stream().str().c_str()); - - CHECK( - logValue.GetParseError() == - rapidjson::ParseErrorCode::kParseErrorNone); + boost::system::error_code ec; + auto logValue = boost::json::parse(stream().str(), ec); + CHECK(ec == boost::system::errc::success); CHECK( - logValue["MessageParams"]["Level"].GetString() == + logValue.as_object()["MessageParams"].as_object()["Level"].get_string() == beast::severities::to_string(beast::severities::kFatal)); } } @@ -445,14 +433,12 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogStream") { journal().stream(beast::severities::kError) << "Test"; - rapidjson::Document logValue; - logValue.Parse(stream().str().c_str()); + boost::system::error_code ec; + auto logValue = boost::json::parse(stream().str(), ec); + CHECK(ec == boost::system::errc::success); CHECK( - logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); - - CHECK( - logValue["MessageParams"]["Level"].GetString() == + logValue.as_object()["MessageParams"].as_object()["Level"].get_string() == beast::severities::to_string(beast::severities::kError)); } @@ -461,25 +447,28 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogParams") journal().debug() << "Test: " << log::param("Field1", 1) << ", " << log::param( "Field2", - std::numeric_limits::max()); - - rapidjson::Document logValue; - logValue.Parse(stream().str().c_str()); - - CHECK( - logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); - - CHECK(logValue["MessageParams"].IsObject()); - CHECK(logValue["MessageParams"]["Field1"].IsNumber()); - CHECK(logValue["MessageParams"]["Field1"].GetInt() == 1); - CHECK(logValue["MessageParams"]["Field2"].IsNumber()); + std::numeric_limits::max()) + << ", " + << log::param("Field3", std::numbers::pi); + + boost::system::error_code ec; + auto logValue = boost::json::parse(stream().str(), ec); + CHECK(ec == boost::system::errc::success); + + CHECK(logValue.as_object()["MessageParams"].is_object()); + CHECK(logValue.as_object()["MessageParams"].as_object()["Field1"].is_number()); + CHECK(logValue.as_object()["MessageParams"].as_object()["Field1"].get_int64() == 1); + CHECK(logValue.as_object()["MessageParams"].as_object()["Field2"].is_number()); CHECK( - logValue["MessageParams"]["Field2"].GetUint64() == + logValue.as_object()["MessageParams"].as_object()["Field2"].get_uint64() == std::numeric_limits::max()); - CHECK(logValue["Message"].IsString()); CHECK( - logValue["Message"].GetString() == - std::string{"Test: 1, 18446744073709551615"}); + logValue.as_object()["MessageParams"].as_object()["Field3"].get_double() == + 3.141593); + CHECK(logValue.as_object()["Message"].is_string()); + CHECK( + logValue.as_object()["Message"].get_string() == + std::string{"Test: 1, 18446744073709551615, 3.141593"}); } TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") @@ -489,24 +478,22 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") "Field2", std::numeric_limits::max()); - rapidjson::Document logValue; - logValue.Parse(stream().str().c_str()); + boost::system::error_code ec; + auto logValue = boost::json::parse(stream().str(), ec); + CHECK(ec == boost::system::errc::success); - CHECK( - logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); - - CHECK(logValue["MessageParams"].IsObject()); - CHECK(logValue["MessageParams"]["Field1"].IsNumber()); - CHECK(logValue["MessageParams"]["Field1"].GetInt() == 1); + CHECK(logValue.as_object()["MessageParams"].is_object()); + CHECK(logValue.as_object()["MessageParams"].as_object()["Field1"].is_number()); + CHECK(logValue.as_object()["MessageParams"].as_object()["Field1"].get_int64() == 1); // UInt64 doesn't fit in Json::Value so it should be converted to a string // NOTE: We should expect it to be an int64 after we make the json library // support in64 and uint64 - CHECK(logValue["MessageParams"]["Field2"].IsNumber()); + CHECK(logValue.as_object()["MessageParams"].as_object()["Field2"].is_number()); CHECK( - logValue["MessageParams"]["Field2"].GetUint64() == + logValue.as_object()["MessageParams"].as_object()["Field2"].get_uint64() == std::numeric_limits::max()); - CHECK(logValue["Message"].IsString()); - CHECK(logValue["Message"].GetString() == std::string{"Test"}); + CHECK(logValue.as_object()["Message"].is_string()); + CHECK(logValue.as_object()["Message"].get_string() == "Test"); } TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributes") @@ -517,18 +504,16 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributes") j.debug() << "Test"; - rapidjson::Document logValue; - logValue.Parse(stream().str().c_str()); - - CHECK( - logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + boost::system::error_code ec; + auto logValue = boost::json::parse(stream().str(), ec); + CHECK(ec == boost::system::errc::success); - CHECK(logValue["JournalParams"]["Field1"].IsString()); + CHECK(logValue.as_object()["JournalParams"].as_object()["Field1"].is_string()); CHECK( - logValue["JournalParams"]["Field1"].GetString() == + logValue.as_object()["JournalParams"].as_object()["Field1"].get_string() == std::string{"Value1"}); - CHECK(logValue["JournalParams"]["Field2"].IsNumber()); - CHECK(logValue["JournalParams"]["Field2"].GetInt() == 2); + CHECK(logValue.as_object()["JournalParams"].as_object()["Field2"].is_number()); + CHECK(logValue.as_object()["JournalParams"].as_object()["Field2"].get_int64() == 2); } TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributesInheritable") @@ -540,22 +525,20 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributesInheritable") j2.debug() << "Test"; - rapidjson::Document logValue; - logValue.Parse(stream().str().c_str()); + boost::system::error_code ec; + auto logValue = boost::json::parse(stream().str(), ec); + CHECK(ec == boost::system::errc::success); + CHECK(logValue.as_object()["JournalParams"].as_object()["Field1"].is_string()); CHECK( - logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); - - CHECK(logValue["JournalParams"]["Field1"].IsString()); - CHECK( - logValue["JournalParams"]["Field1"].GetString() == + logValue.as_object()["JournalParams"].as_object()["Field1"].get_string() == std::string{"Value1"}); - CHECK(logValue["JournalParams"]["Field3"].IsString()); + CHECK(logValue.as_object()["JournalParams"].as_object()["Field3"].is_string()); CHECK( - logValue["JournalParams"]["Field3"].GetString() == + logValue.as_object()["JournalParams"].as_object()["Field3"].get_string() == std::string{"Value3"}); - CHECK(logValue["JournalParams"]["Field2"].IsNumber()); - CHECK(logValue["JournalParams"]["Field2"].GetInt() == 2); + CHECK(logValue.as_object()["JournalParams"].as_object()["Field2"].is_number()); + CHECK(logValue.as_object()["JournalParams"].as_object()["Field2"].get_int64() == 2); } TEST_CASE_FIXTURE( @@ -569,23 +552,21 @@ TEST_CASE_FIXTURE( j2.debug() << "Test"; - rapidjson::Document logValue; - logValue.Parse(stream().str().c_str()); + boost::system::error_code ec; + auto logValue = boost::json::parse(stream().str(), ec); + CHECK(ec == boost::system::errc::success); + CHECK(logValue.as_object()["JournalParams"].as_object()["Field1"].is_string()); CHECK( - logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); - - CHECK(logValue["JournalParams"]["Field1"].IsString()); - CHECK( - logValue["JournalParams"]["Field1"].GetString() == + logValue.as_object()["JournalParams"].as_object()["Field1"].get_string() == std::string{"Value1"}); - CHECK(logValue["JournalParams"]["Field3"].IsString()); + CHECK(logValue.as_object()["JournalParams"].as_object()["Field3"].is_string()); CHECK( - logValue["JournalParams"]["Field3"].GetString() == + logValue.as_object()["JournalParams"].as_object()["Field3"].get_string() == std::string{"Value3"}); // Field2 should be overwritten to 0 - CHECK(logValue["JournalParams"]["Field2"].IsNumber()); - CHECK(logValue["JournalParams"]["Field2"].GetInt() == 2); + CHECK(logValue.as_object()["JournalParams"].as_object()["Field2"].is_number()); + CHECK(logValue.as_object()["JournalParams"].as_object()["Field2"].get_int64() == 2); } TEST_CASE_FIXTURE( @@ -602,18 +583,16 @@ TEST_CASE_FIXTURE( j2.debug() << "Test"; - rapidjson::Document logValue; - logValue.Parse(stream().str().c_str()); - - CHECK( - logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + boost::system::error_code ec; + auto logValue = boost::json::parse(stream().str(), ec); + CHECK(ec == boost::system::errc::success); - CHECK(logValue["JournalParams"]["Field1"].IsString()); + CHECK(logValue.as_object()["JournalParams"].as_object()["Field1"].is_string()); CHECK( - logValue["JournalParams"]["Field1"].GetString() == + logValue.as_object()["JournalParams"].as_object()["Field1"].get_string() == std::string{"Value1"}); - CHECK(logValue["JournalParams"]["Field2"].IsNumber()); - CHECK(logValue["JournalParams"]["Field2"].GetInt() == 2); + CHECK(logValue.as_object()["JournalParams"].as_object()["Field2"].is_number()); + CHECK(logValue.as_object()["JournalParams"].as_object()["Field2"].get_int64() == 2); } TEST_CASE_FIXTURE( @@ -630,16 +609,14 @@ TEST_CASE_FIXTURE( j2.debug() << "Test"; - rapidjson::Document logValue; - logValue.Parse(stream().str().c_str()); - - CHECK( - logValue.GetParseError() == rapidjson::ParseErrorCode::kParseErrorNone); + boost::system::error_code ec; + auto logValue = boost::json::parse(stream().str(), ec); + CHECK(ec == boost::system::errc::success); - CHECK(logValue["JournalParams"]["Field1"].IsString()); + CHECK(logValue.as_object()["JournalParams"].as_object()["Field1"].is_string()); CHECK( - logValue["JournalParams"]["Field1"].GetString() == + logValue.as_object()["JournalParams"].as_object()["Field1"].get_string() == std::string{"Value1"}); - CHECK(logValue["JournalParams"]["Field2"].IsNumber()); - CHECK(logValue["JournalParams"]["Field2"].GetInt() == 2); + CHECK(logValue.as_object()["JournalParams"].as_object()["Field2"].is_number()); + CHECK(logValue.as_object()["JournalParams"].as_object()["Field2"].get_int64() == 2); } \ No newline at end of file From 3b2edce8136703a4734b837377b3726df368b4d7 Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 2 Sep 2025 17:29:16 +0100 Subject: [PATCH 37/88] Fix formatting Signed-off-by: JCW --- src/tests/libxrpl/basics/log.cpp | 227 ++++++++++++++++++++++--------- 1 file changed, 161 insertions(+), 66 deletions(-) diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index 9bdb6f49ec6..882459b6919 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -19,11 +19,12 @@ #include -#include #include -#include +#include + #include +#include using namespace ripple; @@ -178,9 +179,12 @@ TEST_CASE("Global attributes") CHECK(jsonLog.as_object().contains("GlobalParams")); CHECK(jsonLog.as_object()["GlobalParams"].is_object()); CHECK(jsonLog.as_object()["GlobalParams"].as_object().contains("Field1")); - CHECK(jsonLog.as_object()["GlobalParams"].as_object()["Field1"].is_string()); CHECK( - jsonLog.as_object()["GlobalParams"].as_object()["Field1"].get_string() == "Value1"); + jsonLog.as_object()["GlobalParams"].as_object()["Field1"].is_string()); + CHECK( + jsonLog.as_object()["GlobalParams"] + .as_object()["Field1"] + .get_string() == "Value1"); beast::Journal::disableStructuredJournal(); } @@ -207,13 +211,20 @@ TEST_CASE("Global attributes inheritable") CHECK(jsonLog.is_object()); CHECK(jsonLog.as_object()["GlobalParams"].as_object().contains("Field1")); - CHECK(jsonLog.as_object()["GlobalParams"].as_object()["Field1"].is_string()); CHECK( - jsonLog.as_object()["GlobalParams"].as_object()["Field1"].get_string() == "Value1"); + jsonLog.as_object()["GlobalParams"].as_object()["Field1"].is_string()); + CHECK( + jsonLog.as_object()["GlobalParams"] + .as_object()["Field1"] + .get_string() == "Value1"); CHECK( - jsonLog.as_object()["JournalParams"].as_object()["Field1"].get_string() == "Value3"); + jsonLog.as_object()["JournalParams"] + .as_object()["Field1"] + .get_string() == "Value3"); CHECK( - jsonLog.as_object()["JournalParams"].as_object()["Field2"].get_string() == "Value2"); + jsonLog.as_object()["JournalParams"] + .as_object()["Field2"] + .get_string() == "Value2"); beast::Journal::disableStructuredJournal(); } @@ -334,18 +345,26 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") CHECK(logValue.as_object()["MessageParams"].is_object()); CHECK(logValue.as_object()["Message"].is_string()); - CHECK(logValue.as_object()["MessageParams"].as_object().contains("Function")); + CHECK( + logValue.as_object()["MessageParams"].as_object().contains("Function")); CHECK(logValue.as_object()["MessageParams"].as_object().contains("File")); CHECK(logValue.as_object()["MessageParams"].as_object().contains("Line")); - CHECK(logValue.as_object()["MessageParams"].as_object().contains("ThreadId")); + CHECK( + logValue.as_object()["MessageParams"].as_object().contains("ThreadId")); CHECK(logValue.as_object()["MessageParams"].as_object().contains("Level")); CHECK(logValue.as_object()["MessageParams"].as_object().contains("Time")); - CHECK(logValue.as_object()["MessageParams"].as_object()["Function"].is_string()); - CHECK(logValue.as_object()["MessageParams"].as_object()["File"].is_string()); - CHECK(logValue.as_object()["MessageParams"].as_object()["Line"].is_number()); + CHECK(logValue.as_object()["MessageParams"] + .as_object()["Function"] + .is_string()); + CHECK( + logValue.as_object()["MessageParams"].as_object()["File"].is_string()); + CHECK( + logValue.as_object()["MessageParams"].as_object()["Line"].is_number()); - CHECK(logValue.as_object()["Message"].get_string() == std::string{"true Test false"}); + CHECK( + logValue.as_object()["Message"].get_string() == + std::string{"true Test false"}); } TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") @@ -359,7 +378,9 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["MessageParams"].as_object()["Level"].get_string() == + logValue.as_object()["MessageParams"] + .as_object()["Level"] + .get_string() == beast::severities::to_string(beast::severities::kTrace)); } @@ -372,7 +393,9 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["MessageParams"].as_object()["Level"].get_string() == + logValue.as_object()["MessageParams"] + .as_object()["Level"] + .get_string() == beast::severities::to_string(beast::severities::kDebug)); } @@ -385,7 +408,9 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["MessageParams"].as_object()["Level"].get_string() == + logValue.as_object()["MessageParams"] + .as_object()["Level"] + .get_string() == beast::severities::to_string(beast::severities::kInfo)); } @@ -398,7 +423,9 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["MessageParams"].as_object()["Level"].get_string() == + logValue.as_object()["MessageParams"] + .as_object()["Level"] + .get_string() == beast::severities::to_string(beast::severities::kWarning)); } @@ -411,7 +438,9 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["MessageParams"].as_object()["Level"].get_string() == + logValue.as_object()["MessageParams"] + .as_object()["Level"] + .get_string() == beast::severities::to_string(beast::severities::kError)); } @@ -424,7 +453,9 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["MessageParams"].as_object()["Level"].get_string() == + logValue.as_object()["MessageParams"] + .as_object()["Level"] + .get_string() == beast::severities::to_string(beast::severities::kFatal)); } } @@ -438,7 +469,9 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogStream") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["MessageParams"].as_object()["Level"].get_string() == + logValue.as_object()["MessageParams"] + .as_object()["Level"] + .get_string() == beast::severities::to_string(beast::severities::kError)); } @@ -448,23 +481,31 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogParams") << log::param( "Field2", std::numeric_limits::max()) - << ", " - << log::param("Field3", std::numbers::pi); + << ", " << log::param("Field3", std::numbers::pi); boost::system::error_code ec; auto logValue = boost::json::parse(stream().str(), ec); CHECK(ec == boost::system::errc::success); CHECK(logValue.as_object()["MessageParams"].is_object()); - CHECK(logValue.as_object()["MessageParams"].as_object()["Field1"].is_number()); - CHECK(logValue.as_object()["MessageParams"].as_object()["Field1"].get_int64() == 1); - CHECK(logValue.as_object()["MessageParams"].as_object()["Field2"].is_number()); + CHECK(logValue.as_object()["MessageParams"] + .as_object()["Field1"] + .is_number()); CHECK( - logValue.as_object()["MessageParams"].as_object()["Field2"].get_uint64() == - std::numeric_limits::max()); + logValue.as_object()["MessageParams"] + .as_object()["Field1"] + .get_int64() == 1); + CHECK(logValue.as_object()["MessageParams"] + .as_object()["Field2"] + .is_number()); CHECK( - logValue.as_object()["MessageParams"].as_object()["Field3"].get_double() == - 3.141593); + logValue.as_object()["MessageParams"] + .as_object()["Field2"] + .get_uint64() == std::numeric_limits::max()); + CHECK( + logValue.as_object()["MessageParams"] + .as_object()["Field3"] + .get_double() == 3.141593); CHECK(logValue.as_object()["Message"].is_string()); CHECK( logValue.as_object()["Message"].get_string() == @@ -483,15 +524,23 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") CHECK(ec == boost::system::errc::success); CHECK(logValue.as_object()["MessageParams"].is_object()); - CHECK(logValue.as_object()["MessageParams"].as_object()["Field1"].is_number()); - CHECK(logValue.as_object()["MessageParams"].as_object()["Field1"].get_int64() == 1); + CHECK(logValue.as_object()["MessageParams"] + .as_object()["Field1"] + .is_number()); + CHECK( + logValue.as_object()["MessageParams"] + .as_object()["Field1"] + .get_int64() == 1); // UInt64 doesn't fit in Json::Value so it should be converted to a string // NOTE: We should expect it to be an int64 after we make the json library // support in64 and uint64 - CHECK(logValue.as_object()["MessageParams"].as_object()["Field2"].is_number()); + CHECK(logValue.as_object()["MessageParams"] + .as_object()["Field2"] + .is_number()); CHECK( - logValue.as_object()["MessageParams"].as_object()["Field2"].get_uint64() == - std::numeric_limits::max()); + logValue.as_object()["MessageParams"] + .as_object()["Field2"] + .get_uint64() == std::numeric_limits::max()); CHECK(logValue.as_object()["Message"].is_string()); CHECK(logValue.as_object()["Message"].get_string() == "Test"); } @@ -508,12 +557,20 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributes") auto logValue = boost::json::parse(stream().str(), ec); CHECK(ec == boost::system::errc::success); - CHECK(logValue.as_object()["JournalParams"].as_object()["Field1"].is_string()); + CHECK(logValue.as_object()["JournalParams"] + .as_object()["Field1"] + .is_string()); CHECK( - logValue.as_object()["JournalParams"].as_object()["Field1"].get_string() == - std::string{"Value1"}); - CHECK(logValue.as_object()["JournalParams"].as_object()["Field2"].is_number()); - CHECK(logValue.as_object()["JournalParams"].as_object()["Field2"].get_int64() == 2); + logValue.as_object()["JournalParams"] + .as_object()["Field1"] + .get_string() == std::string{"Value1"}); + CHECK(logValue.as_object()["JournalParams"] + .as_object()["Field2"] + .is_number()); + CHECK( + logValue.as_object()["JournalParams"] + .as_object()["Field2"] + .get_int64() == 2); } TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributesInheritable") @@ -529,16 +586,27 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributesInheritable") auto logValue = boost::json::parse(stream().str(), ec); CHECK(ec == boost::system::errc::success); - CHECK(logValue.as_object()["JournalParams"].as_object()["Field1"].is_string()); + CHECK(logValue.as_object()["JournalParams"] + .as_object()["Field1"] + .is_string()); + CHECK( + logValue.as_object()["JournalParams"] + .as_object()["Field1"] + .get_string() == std::string{"Value1"}); + CHECK(logValue.as_object()["JournalParams"] + .as_object()["Field3"] + .is_string()); CHECK( - logValue.as_object()["JournalParams"].as_object()["Field1"].get_string() == - std::string{"Value1"}); - CHECK(logValue.as_object()["JournalParams"].as_object()["Field3"].is_string()); + logValue.as_object()["JournalParams"] + .as_object()["Field3"] + .get_string() == std::string{"Value3"}); + CHECK(logValue.as_object()["JournalParams"] + .as_object()["Field2"] + .is_number()); CHECK( - logValue.as_object()["JournalParams"].as_object()["Field3"].get_string() == - std::string{"Value3"}); - CHECK(logValue.as_object()["JournalParams"].as_object()["Field2"].is_number()); - CHECK(logValue.as_object()["JournalParams"].as_object()["Field2"].get_int64() == 2); + logValue.as_object()["JournalParams"] + .as_object()["Field2"] + .get_int64() == 2); } TEST_CASE_FIXTURE( @@ -556,17 +624,28 @@ TEST_CASE_FIXTURE( auto logValue = boost::json::parse(stream().str(), ec); CHECK(ec == boost::system::errc::success); - CHECK(logValue.as_object()["JournalParams"].as_object()["Field1"].is_string()); + CHECK(logValue.as_object()["JournalParams"] + .as_object()["Field1"] + .is_string()); CHECK( - logValue.as_object()["JournalParams"].as_object()["Field1"].get_string() == - std::string{"Value1"}); - CHECK(logValue.as_object()["JournalParams"].as_object()["Field3"].is_string()); + logValue.as_object()["JournalParams"] + .as_object()["Field1"] + .get_string() == std::string{"Value1"}); + CHECK(logValue.as_object()["JournalParams"] + .as_object()["Field3"] + .is_string()); CHECK( - logValue.as_object()["JournalParams"].as_object()["Field3"].get_string() == - std::string{"Value3"}); + logValue.as_object()["JournalParams"] + .as_object()["Field3"] + .get_string() == std::string{"Value3"}); // Field2 should be overwritten to 0 - CHECK(logValue.as_object()["JournalParams"].as_object()["Field2"].is_number()); - CHECK(logValue.as_object()["JournalParams"].as_object()["Field2"].get_int64() == 2); + CHECK(logValue.as_object()["JournalParams"] + .as_object()["Field2"] + .is_number()); + CHECK( + logValue.as_object()["JournalParams"] + .as_object()["Field2"] + .get_int64() == 2); } TEST_CASE_FIXTURE( @@ -587,12 +666,20 @@ TEST_CASE_FIXTURE( auto logValue = boost::json::parse(stream().str(), ec); CHECK(ec == boost::system::errc::success); - CHECK(logValue.as_object()["JournalParams"].as_object()["Field1"].is_string()); + CHECK(logValue.as_object()["JournalParams"] + .as_object()["Field1"] + .is_string()); CHECK( - logValue.as_object()["JournalParams"].as_object()["Field1"].get_string() == - std::string{"Value1"}); - CHECK(logValue.as_object()["JournalParams"].as_object()["Field2"].is_number()); - CHECK(logValue.as_object()["JournalParams"].as_object()["Field2"].get_int64() == 2); + logValue.as_object()["JournalParams"] + .as_object()["Field1"] + .get_string() == std::string{"Value1"}); + CHECK(logValue.as_object()["JournalParams"] + .as_object()["Field2"] + .is_number()); + CHECK( + logValue.as_object()["JournalParams"] + .as_object()["Field2"] + .get_int64() == 2); } TEST_CASE_FIXTURE( @@ -613,10 +700,18 @@ TEST_CASE_FIXTURE( auto logValue = boost::json::parse(stream().str(), ec); CHECK(ec == boost::system::errc::success); - CHECK(logValue.as_object()["JournalParams"].as_object()["Field1"].is_string()); + CHECK(logValue.as_object()["JournalParams"] + .as_object()["Field1"] + .is_string()); + CHECK( + logValue.as_object()["JournalParams"] + .as_object()["Field1"] + .get_string() == std::string{"Value1"}); + CHECK(logValue.as_object()["JournalParams"] + .as_object()["Field2"] + .is_number()); CHECK( - logValue.as_object()["JournalParams"].as_object()["Field1"].get_string() == - std::string{"Value1"}); - CHECK(logValue.as_object()["JournalParams"].as_object()["Field2"].is_number()); - CHECK(logValue.as_object()["JournalParams"].as_object()["Field2"].get_int64() == 2); + logValue.as_object()["JournalParams"] + .as_object()["Field2"] + .get_int64() == 2); } \ No newline at end of file From 67aa3d5ac95e353fb39f37e077e081f1b280b852 Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 2 Sep 2025 17:36:45 +0100 Subject: [PATCH 38/88] Remove hardcoded logstyle Signed-off-by: JCW --- src/xrpld/core/detail/Config.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/xrpld/core/detail/Config.cpp b/src/xrpld/core/detail/Config.cpp index 726721abd46..7eb3a68a46d 100644 --- a/src/xrpld/core/detail/Config.cpp +++ b/src/xrpld/core/detail/Config.cpp @@ -690,7 +690,8 @@ Config::loadFromString(std::string const& fileContents) if (getSingleSection(secConfig, SECTION_DEBUG_LOGFILE, strTemp, j_)) DEBUG_LOGFILE = strTemp; - LOG_STYLE = LogStyle::Json; + if (getSingleSection(secConfig, SECTION_LOG_STYLE, strTemp, j_)) + LOG_STYLE = LogStyle::fromString(strTemp); if (getSingleSection(secConfig, SECTION_SWEEP_INTERVAL, strTemp, j_)) { From 89ebb6b49591d872af06fb59ec3062e89dd2cbd8 Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 2 Sep 2025 18:17:19 +0100 Subject: [PATCH 39/88] Fix issues Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 76 ++++++++++++++++++++-------- src/xrpld/app/main/Application.cpp | 4 +- src/xrpld/app/main/Application.h | 2 +- 3 files changed, 59 insertions(+), 23 deletions(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 5723006f093..0e26164ecec 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -79,6 +79,16 @@ operator<<(std::ostream& os, LogParameter const& param); namespace beast { +// Forward declaration for use in interfaces +namespace detail { +class SimpleJsonWriter; +} + +// Type alias for journal attribute factory functions +using JournalAttributesFactory = void(detail::SimpleJsonWriter&); + +namespace detail { + class SimpleJsonWriter { public: @@ -125,6 +135,16 @@ class SimpleJsonWriter stream_.append("\","sv); } std::string_view + writeInt(std::int8_t val) const + { + return pushNumber(val, stream_); + } + std::string_view + writeInt(std::int16_t val) const + { + return pushNumber(val, stream_); + } + std::string_view writeInt(std::int32_t val) const { return pushNumber(val, stream_); @@ -135,6 +155,21 @@ class SimpleJsonWriter return pushNumber(val, stream_); } std::string_view + writeUInt(std::size_t val) const + { + return pushNumber(val, stream_); + } + std::string_view + writeUInt(std::uint8_t val) const + { + return pushNumber(val, stream_); + } + std::string_view + writeUInt(std::uint16_t val) const + { + return pushNumber(val, stream_); + } + std::string_view writeUInt(std::uint32_t val) const { return pushNumber(val, stream_); @@ -259,6 +294,8 @@ class SimpleJsonWriter std::string& stream_; }; +} // namespace detail + /** A namespace for easy access to logging severity values. */ namespace severities { /** Severity level / threshold of a Journal message. */ @@ -312,7 +349,7 @@ class Journal class JsonLogContext { std::string buffer_; - SimpleJsonWriter messageParamsWriter_; + detail::SimpleJsonWriter messageParamsWriter_; public: JsonLogContext() : messageParamsWriter_(buffer_) @@ -320,7 +357,7 @@ class Journal buffer_.reserve(1024 * 5); } - SimpleJsonWriter& + detail::SimpleJsonWriter& writer() { return messageParamsWriter_; @@ -596,7 +633,7 @@ class Journal : m_name(other.m_name), m_sink(other.m_sink) { std::string stream{other.m_attributesJson}; - SimpleJsonWriter writer{stream}; + detail::SimpleJsonWriter writer{stream}; if (other.m_attributesJson.empty()) { writer.startObject(); @@ -620,7 +657,7 @@ class Journal : m_name(name), m_sink(&sink) { std::string stream; - SimpleJsonWriter writer{stream}; + detail::SimpleJsonWriter writer{stream}; writer.startObject(); attributesFactory(writer); m_attributesJson = std::move(stream); @@ -741,7 +778,7 @@ class Journal auto isEmpty = globalLogAttributesJson_.empty(); std::string stream{std::move(globalLogAttributesJson_)}; - SimpleJsonWriter writer{stream}; + detail::SimpleJsonWriter writer{stream}; if (isEmpty) { writer.startObject(); @@ -860,7 +897,7 @@ namespace ripple::log { namespace detail { template -concept CanToChars = requires(T val) { +concept ToCharsFormattable = requires(T val) { { to_chars(std::declval(), std::declval(), val) } -> std::convertible_to; @@ -869,14 +906,22 @@ concept CanToChars = requires(T val) { template void setJsonValue( - beast::SimpleJsonWriter& writer, + beast::detail::SimpleJsonWriter& writer, char const* name, T&& value, std::ostream* outStream) { using ValueType = std::decay_t; writer.writeKey(name); - if constexpr (std::is_integral_v) + if constexpr (std::is_same_v) + { + auto sv = writer.writeBool(value); + if (outStream) + { + outStream->write(sv.data(), sv.size()); + } + } + else if constexpr (std::is_integral_v) { std::string_view sv; if constexpr (std::is_signed_v) @@ -901,14 +946,6 @@ setJsonValue( outStream->write(sv.data(), sv.size()); } } - else if constexpr (std::is_same_v) - { - auto sv = writer.writeBool(value); - if (outStream) - { - outStream->write(sv.data(), sv.size()); - } - } else if constexpr ( std::is_same_v || std::is_same_v) @@ -929,15 +966,14 @@ setJsonValue( } else { - if constexpr (CanToChars) + if constexpr (ToCharsFormattable) { char buffer[1024]; std::to_chars_result result = to_chars(std::begin(buffer), std::end(buffer), value); if (result.ec == std::errc{}) { - std::string_view sv; - sv = {std::begin(buffer), result.ptr}; + std::string_view sv{std::begin(buffer), result.ptr}; writer.writeString(sv); if (outStream) { @@ -1010,7 +1046,7 @@ template [[nodiscard]] auto attributes(Pair&&... pairs) { - return [&](beast::SimpleJsonWriter& writer) { + return [&](beast::detail::SimpleJsonWriter& writer) { (detail::setJsonValue(writer, pairs.first, pairs.second, nullptr), ...); }; } diff --git a/src/xrpld/app/main/Application.cpp b/src/xrpld/app/main/Application.cpp index 98f48f66578..82f86e789e9 100644 --- a/src/xrpld/app/main/Application.cpp +++ b/src/xrpld/app/main/Application.cpp @@ -834,7 +834,7 @@ class ApplicationImp : public Application, public BasicApp beast::Journal journal( std::string const& name, - std::function const& attributes) + std::function const& attributes) override; beast::Journal @@ -2178,7 +2178,7 @@ ApplicationImp::serverOkay(std::string& reason) beast::Journal ApplicationImp::journal( std::string const& name, - std::function const& attributes) + std::function const& attributes) { return logs_->journal(name, std::move(attributes)); } diff --git a/src/xrpld/app/main/Application.h b/src/xrpld/app/main/Application.h index d2127285927..03926f27ea4 100644 --- a/src/xrpld/app/main/Application.h +++ b/src/xrpld/app/main/Application.h @@ -260,7 +260,7 @@ class Application : public beast::PropertyStream::Source virtual beast::Journal journal( std::string const& name, - std::function const& attributes) = 0; + std::function const& attributes) = 0; virtual beast::Journal journal(std::string const& name) = 0; From addfae12137f035d97b207aceb01f0d42281bd96 Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 2 Sep 2025 18:19:58 +0100 Subject: [PATCH 40/88] Fix formatting Signed-off-by: JCW --- src/tests/libxrpl/basics/log.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index 882459b6919..b8504b6219c 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -714,4 +714,4 @@ TEST_CASE_FIXTURE( logValue.as_object()["JournalParams"] .as_object()["Field2"] .get_int64() == 2); -} \ No newline at end of file +} From bd7b098409aea2fb66862d9dac12cbb025890c54 Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 2 Sep 2025 18:23:21 +0100 Subject: [PATCH 41/88] Fix levelisation Signed-off-by: JCW --- .github/scripts/levelization/results/ordering.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/scripts/levelization/results/ordering.txt b/.github/scripts/levelization/results/ordering.txt index 7bc289c5c88..82e2bf24a6c 100644 --- a/.github/scripts/levelization/results/ordering.txt +++ b/.github/scripts/levelization/results/ordering.txt @@ -62,7 +62,6 @@ test.csf > xrpl.basics test.csf > xrpld.consensus test.csf > xrpl.json test.csf > xrpl.protocol -test.csf > xrpl.telemetry test.json > test.jtx test.json > xrpl.json test.jtx > xrpl.basics @@ -148,12 +147,9 @@ xrpl.protocol > xrpl.json xrpl.resource > xrpl.basics xrpl.resource > xrpl.json xrpl.resource > xrpl.protocol -xrpl.resource > xrpl.telemetry xrpl.server > xrpl.basics xrpl.server > xrpl.json xrpl.server > xrpl.protocol -xrpl.server > xrpl.telemetry -xrpl.telemetry > xrpl.json xrpld.app > test.unit_test xrpld.app > xrpl.basics xrpld.app > xrpld.conditions @@ -165,7 +161,6 @@ xrpld.app > xrpl.ledger xrpld.app > xrpl.net xrpld.app > xrpl.protocol xrpld.app > xrpl.resource -xrpld.app > xrpl.telemetry xrpld.conditions > xrpl.basics xrpld.conditions > xrpl.protocol xrpld.consensus > xrpl.basics @@ -175,7 +170,6 @@ xrpld.core > xrpl.basics xrpld.core > xrpl.json xrpld.core > xrpl.net xrpld.core > xrpl.protocol -xrpld.core > xrpl.telemetry xrpld.ledger > xrpl.basics xrpld.ledger > xrpl.json xrpld.ledger > xrpl.protocol From cd8d5d97d1c5e057b0bc91bff2551417815d1fb6 Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 2 Sep 2025 19:09:26 +0100 Subject: [PATCH 42/88] Fix issues Signed-off-by: JCW --- src/tests/libxrpl/basics/log.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index b8504b6219c..7651d218e57 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -232,7 +232,7 @@ TEST_CASE("Test JsonWriter") { { std::string stream; - beast::SimpleJsonWriter writer{stream}; + beast::detail::SimpleJsonWriter writer{stream}; writer.writeString("\n"); CHECK(writer.finish() == "\"\\n\""); @@ -240,7 +240,7 @@ TEST_CASE("Test JsonWriter") { std::string stream; - beast::SimpleJsonWriter writer{stream}; + beast::detail::SimpleJsonWriter writer{stream}; writer.writeString("\t"); CHECK(writer.finish() == "\"\\t\""); @@ -248,7 +248,7 @@ TEST_CASE("Test JsonWriter") { std::string stream; - beast::SimpleJsonWriter writer{stream}; + beast::detail::SimpleJsonWriter writer{stream}; writer.writeString(std::string_view{"\0", 1}); CHECK(writer.finish() == "\"\\u0000\""); @@ -256,7 +256,7 @@ TEST_CASE("Test JsonWriter") { std::string stream; - beast::SimpleJsonWriter writer{stream}; + beast::detail::SimpleJsonWriter writer{stream}; writer.writeString("\"\\"); CHECK(writer.finish() == "\"\\\"\\\\\""); From 458bd8a3bd79d24964a513c974c19282c798dfbf Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 2 Sep 2025 19:24:43 +0100 Subject: [PATCH 43/88] Revert unrelated changes Signed-off-by: JCW --- cmake/RippledCore.cmake | 1 - conanfile.py | 2 +- include/xrpl/beast/utility/Journal.h | 2 +- src/libxrpl/beast/utility/beast_Journal.cpp | 19 ++++--- src/test/core/Coroutine_test.cpp | 62 --------------------- src/xrpld/core/ClosureCounter.h | 7 --- src/xrpld/core/Coro.ipp | 10 ---- src/xrpld/core/detail/JobQueue.cpp | 7 +-- 8 files changed, 15 insertions(+), 95 deletions(-) diff --git a/cmake/RippledCore.cmake b/cmake/RippledCore.cmake index 2e41531ba7a..481b6e3cea6 100644 --- a/cmake/RippledCore.cmake +++ b/cmake/RippledCore.cmake @@ -85,7 +85,6 @@ target_link_libraries(xrpl.libxrpl.basics PUBLIC xrpl.libxrpl.beast) add_module(xrpl json) target_link_libraries(xrpl.libxrpl.json PUBLIC xrpl.libxrpl.basics) - add_module(xrpl crypto) target_link_libraries(xrpl.libxrpl.crypto PUBLIC xrpl.libxrpl.basics) diff --git a/conanfile.py b/conanfile.py index e0373ac65e5..3146b887e0d 100644 --- a/conanfile.py +++ b/conanfile.py @@ -29,7 +29,7 @@ class Xrpl(ConanFile): 'nudb/2.0.9', 'openssl/1.1.1w', 'soci/4.0.3', - 'zlib/1.3.1' + 'zlib/1.3.1', ] test_requires = [ diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 0e26164ecec..0ea1f5e291d 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -313,7 +313,7 @@ enum Severity { kNone = kDisabled }; -std::string +std::string_view to_string(Severity severity); } // namespace severities diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index 74a42fb34a1..9c485cdfa82 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -94,29 +94,30 @@ Journal::getNullSink() //------------------------------------------------------------------------------ -std::string +std::string_view severities::to_string(Severity severity) { + using namespace std::string_view_literals; switch (severity) { case kDisabled: - return "disabled"; + return "disabled"sv; case kTrace: - return "trace"; + return "trace"sv; case kDebug: - return "debug"; + return "debug"sv; case kInfo: - return "info"; + return "info"sv; case kWarning: - return "warning"; + return "warning"sv; case kError: - return "error"; + return "error"sv; case kFatal: - return "fatal"; + return "fatal"sv; default: UNREACHABLE("Unexpected severity value!"); } - return ""; + return ""sv; } void diff --git a/src/test/core/Coroutine_test.cpp b/src/test/core/Coroutine_test.cpp index a3eb3c94404..8458da647d4 100644 --- a/src/test/core/Coroutine_test.cpp +++ b/src/test/core/Coroutine_test.cpp @@ -175,74 +175,12 @@ class Coroutine_test : public beast::unit_test::suite BEAST_EXPECT(*lv == -1); } - void - test_yield_and_stop() - { - using namespace std::chrono_literals; - using namespace jtx; - - testcase("yield and stop"); - - Env env(*this, envconfig([](std::unique_ptr cfg) { - cfg->FORCE_MULTI_THREAD = true; - return cfg; - })); - - std::shared_ptr c; - std::mutex mutexStop; - std::mutex mutexYield; - std::condition_variable cond; - std::condition_variable condYield; - bool yielded = false; - bool stopped = false; - - env.app().getJobQueue().postCoro( - jtCLIENT, "Coroutine-Test", [&](auto const& cr) { - c = cr; - { - std::unique_lock lock(mutexYield); - yielded = true; - condYield.notify_all(); - } - c->yield(); - // Just to keep this job alive - std::this_thread::sleep_for(5ms); - }); - std::thread th{[&]() { - std::unique_lock lock(mutexStop); - cond.wait(lock, [&]() { return stopped; }); - // Delay a bit to wait for stop() to be called - std::this_thread::sleep_for(1ms); - c->post(); - }}; - - // Delay a bit to wait for yield() to be called - std::this_thread::sleep_for(1ms); - std::unique_lock lockYield(mutexYield); - condYield.wait(lockYield, [&]() { return yielded; }); - { - std::unique_lock lock(mutexStop); - stopped = true; - cond.notify_all(); - } - env.app().getJobQueue().stop(); - try - { - th.join(); - } - catch (std::exception const& e) - { - } - pass(); - } - void run() override { correct_order(); incorrect_order(); thread_specific_storage(); - // test_yield_and_stop(); } }; diff --git a/src/xrpld/core/ClosureCounter.h b/src/xrpld/core/ClosureCounter.h index ded056412af..92117a91c45 100644 --- a/src/xrpld/core/ClosureCounter.h +++ b/src/xrpld/core/ClosureCounter.h @@ -180,13 +180,6 @@ class ClosureCounter } } - template - Substitute - forceWrap(Closure&& closure) - { - return {*this, std::forward(closure)}; - } - /** Wrap the passed closure with a reference counter. @param closure Closure that accepts Args_t parameters and returns Ret_t. diff --git a/src/xrpld/core/Coro.ipp b/src/xrpld/core/Coro.ipp index 9abf4bbf811..5901e07c684 100644 --- a/src/xrpld/core/Coro.ipp +++ b/src/xrpld/core/Coro.ipp @@ -98,11 +98,6 @@ JobQueue::Coro::resume() } { std::lock_guard lock(jq_.m_mutex); - - XRPL_ASSERT( - jq_.nSuspend_ > 0, - "ripple::JobQueue::Coro::resume jq_.nSuspend_ should be greater " - "than 0"); --jq_.nSuspend_; } auto saved = detail::getLocalValues().release(); @@ -139,11 +134,6 @@ JobQueue::Coro::expectEarlyExit() // That said, since we're outside the Coro's stack, we need to // decrement the nSuspend that the Coro's call to yield caused. std::lock_guard lock(jq_.m_mutex); - - XRPL_ASSERT( - jq_.nSuspend_ > 0, - "ripple::JobQueue::Coro::expectEarlyExit() jq_.nSuspend_ should be " - "greater than 0"); --jq_.nSuspend_; #ifndef NDEBUG finished_ = true; diff --git a/src/xrpld/core/detail/JobQueue.cpp b/src/xrpld/core/detail/JobQueue.cpp index 986bb68545f..1ea1df51abb 100644 --- a/src/xrpld/core/detail/JobQueue.cpp +++ b/src/xrpld/core/detail/JobQueue.cpp @@ -304,10 +304,9 @@ JobQueue::stop() // but there may still be some threads between the return of // `Job::doJob` and the return of `JobQueue::processTask`. That is why // we must wait on the condition variable to make these assertions. - std::unique_lock lock(m_mutex); - cv_.wait(lock, [this] { - return m_processCount == 0 && nSuspend_ == 0 && m_jobSet.empty(); - }); + std::unique_lock lock(m_mutex); + cv_.wait( + lock, [this] { return m_processCount == 0 && m_jobSet.empty(); }); XRPL_ASSERT( m_processCount == 0, "ripple::JobQueue::stop : all processes completed"); From c2aae2d846672818fce23a13e4328492c2f8f546 Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 2 Sep 2025 19:41:26 +0100 Subject: [PATCH 44/88] Optimisation Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 8 -------- src/xrpld/app/main/Application.cpp | 23 ----------------------- src/xrpld/app/main/Application.h | 19 ++++++++++++------- 3 files changed, 12 insertions(+), 38 deletions(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 0ea1f5e291d..a335e23cfb7 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -79,14 +79,6 @@ operator<<(std::ostream& os, LogParameter const& param); namespace beast { -// Forward declaration for use in interfaces -namespace detail { -class SimpleJsonWriter; -} - -// Type alias for journal attribute factory functions -using JournalAttributesFactory = void(detail::SimpleJsonWriter&); - namespace detail { class SimpleJsonWriter diff --git a/src/xrpld/app/main/Application.cpp b/src/xrpld/app/main/Application.cpp index 82f86e789e9..5351a906514 100644 --- a/src/xrpld/app/main/Application.cpp +++ b/src/xrpld/app/main/Application.cpp @@ -831,15 +831,6 @@ class ApplicationImp : public Application, public BasicApp bool serverOkay(std::string& reason) override; - beast::Journal - journal( - std::string const& name, - std::function const& attributes) - override; - - beast::Journal - journal(std::string const& name) override; - //-------------------------------------------------------------------------- bool @@ -2175,20 +2166,6 @@ ApplicationImp::serverOkay(std::string& reason) return true; } -beast::Journal -ApplicationImp::journal( - std::string const& name, - std::function const& attributes) -{ - return logs_->journal(name, std::move(attributes)); -} - -beast::Journal -ApplicationImp::journal(std::string const& name) -{ - return logs_->journal(name); -} - void ApplicationImp::setMaxDisallowedLedger() { diff --git a/src/xrpld/app/main/Application.h b/src/xrpld/app/main/Application.h index 03926f27ea4..474982bddfd 100644 --- a/src/xrpld/app/main/Application.h +++ b/src/xrpld/app/main/Application.h @@ -257,13 +257,18 @@ class Application : public beast::PropertyStream::Source virtual bool serverOkay(std::string& reason) = 0; - virtual beast::Journal - journal( - std::string const& name, - std::function const& attributes) = 0; - - virtual beast::Journal - journal(std::string const& name) = 0; + template + beast::Journal + journal(std::string const& name, TAttributesFactory&& factory) + { + return logs().journal(name, std::forward(factory)); + } + + beast::Journal + journal(std::string const& name) + { + return logs().journal(name); + } /* Returns the number of file descriptors the application needs */ virtual int From 1a2b7e9b94175bc27869d8292aa7d0fbf737b260 Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 2 Sep 2025 20:04:14 +0100 Subject: [PATCH 45/88] Fix issues Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 103 ++++++++++---------- src/libxrpl/beast/utility/beast_Journal.cpp | 18 ++-- src/tests/libxrpl/basics/log.cpp | 16 +-- 3 files changed, 71 insertions(+), 66 deletions(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index a335e23cfb7..2800929785b 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -84,140 +85,140 @@ namespace detail { class SimpleJsonWriter { public: - explicit SimpleJsonWriter(std::string& stream) : stream_(stream) + explicit SimpleJsonWriter(std::string& buffer) : buffer_(buffer) { } void startObject() const { - stream_.push_back('{'); + buffer_.push_back('{'); } void endObject() const { using namespace std::string_view_literals; - stream_.pop_back(); - stream_.append("},"sv); + buffer_.pop_back(); + buffer_.append("},"sv); } void writeKey(std::string_view key) const { writeString(key); - stream_.back() = ':'; + buffer_.back() = ':'; } void startArray() const { - stream_.push_back('['); + buffer_.push_back('['); } void endArray() const { using namespace std::string_view_literals; - stream_.pop_back(); - stream_.append("],"sv); + buffer_.pop_back(); + buffer_.append("],"sv); } void writeString(std::string_view str) const { using namespace std::string_view_literals; - stream_.push_back('"'); - escape(str, stream_); - stream_.append("\","sv); + buffer_.push_back('"'); + escape(str, buffer_); + buffer_.append("\","sv); } std::string_view writeInt(std::int8_t val) const { - return pushNumber(val, stream_); + return pushNumber(val, buffer_); } std::string_view writeInt(std::int16_t val) const { - return pushNumber(val, stream_); + return pushNumber(val, buffer_); } std::string_view writeInt(std::int32_t val) const { - return pushNumber(val, stream_); + return pushNumber(val, buffer_); } std::string_view writeInt(std::int64_t val) const { - return pushNumber(val, stream_); + return pushNumber(val, buffer_); } std::string_view writeUInt(std::size_t val) const { - return pushNumber(val, stream_); + return pushNumber(val, buffer_); } std::string_view writeUInt(std::uint8_t val) const { - return pushNumber(val, stream_); + return pushNumber(val, buffer_); } std::string_view writeUInt(std::uint16_t val) const { - return pushNumber(val, stream_); + return pushNumber(val, buffer_); } std::string_view writeUInt(std::uint32_t val) const { - return pushNumber(val, stream_); + return pushNumber(val, buffer_); } std::string_view writeUInt(std::uint64_t val) const { - return pushNumber(val, stream_); + return pushNumber(val, buffer_); } std::string_view writeDouble(double val) const { - return pushNumber(val, stream_); + return pushNumber(val, buffer_); } std::string_view writeBool(bool val) const { using namespace std::string_view_literals; auto str = val ? "true,"sv : "false,"sv; - stream_.append(str); + buffer_.append(str); return str; } void writeNull() const { using namespace std::string_view_literals; - stream_.append("null,"sv); + buffer_.append("null,"sv); } void writeRaw(std::string_view str) const { - stream_.append(str); + buffer_.append(str); } [[nodiscard]] std::string_view finish() { - return std::string_view{stream_.c_str(), stream_.size() - 1}; + return std::string_view{buffer_.c_str(), buffer_.size() - 1}; } private: template static std::string_view - pushNumber(T val, std::string& stream) + pushNumber(T val, std::string& str) { thread_local char buffer[128]; auto result = std::to_chars(std::begin(buffer), std::end(buffer), val); auto ptr = result.ptr; *ptr = ','; auto len = ptr - std::begin(buffer); - stream.append(buffer, len + 1); + str.append(buffer, len + 1); return {buffer, static_cast(len)}; } static void - escape(std::string_view str, std::string& os) + escape(std::string_view str, std::string& buffer) { static constexpr char HEX[] = "0123456789ABCDEF"; @@ -240,36 +241,36 @@ class SimpleJsonWriter // Flush the preceding safe run in one go. if (chunk != p) - os.append(chunk, p - chunk); + buffer.append(chunk, p - chunk); switch (c) { case '"': - os.append("\\\"", 2); + buffer.append("\\\"", 2); break; case '\\': - os.append("\\\\", 2); + buffer.append("\\\\", 2); break; case '\b': - os.append("\\b", 2); + buffer.append("\\b", 2); break; case '\f': - os.append("\\f", 2); + buffer.append("\\f", 2); break; case '\n': - os.append("\\n", 2); + buffer.append("\\n", 2); break; case '\r': - os.append("\\r", 2); + buffer.append("\\r", 2); break; case '\t': - os.append("\\t", 2); + buffer.append("\\t", 2); break; default: { // Other C0 controls -> \u00XX (JSON compliant) char buf[6]{ '\\', 'u', '0', '0', HEX[(c >> 4) & 0xF], HEX[c & 0xF]}; - os.append(buf, 6); + buffer.append(buf, 6); break; } } @@ -280,10 +281,10 @@ class SimpleJsonWriter // Flush trailing safe run if (chunk != p) - os.append(chunk, p - chunk); + buffer.append(chunk, p - chunk); } - std::string& stream_; + std::string& buffer_; }; } // namespace detail @@ -369,7 +370,7 @@ class Journal std::string m_name; std::string m_attributesJson; static std::string globalLogAttributesJson_; - static std::mutex globalLogAttributesMutex_; + static std::shared_mutex globalLogAttributesMutex_; static bool m_jsonLogsEnabled; static thread_local JsonLogContext currentJsonLogContext_; @@ -624,14 +625,14 @@ class Journal Journal(Journal const& other, TAttributesFactory&& attributesFactory) : m_name(other.m_name), m_sink(other.m_sink) { - std::string stream{other.m_attributesJson}; - detail::SimpleJsonWriter writer{stream}; + std::string buffer{other.m_attributesJson}; + detail::SimpleJsonWriter writer{buffer}; if (other.m_attributesJson.empty()) { writer.startObject(); } attributesFactory(writer); - m_attributesJson = std::move(stream); + m_attributesJson = std::move(buffer); } /** Create a journal that writes to the specified sink. */ @@ -648,11 +649,11 @@ class Journal TAttributesFactory&& attributesFactory) : m_name(name), m_sink(&sink) { - std::string stream; - detail::SimpleJsonWriter writer{stream}; + std::string buffer; + detail::SimpleJsonWriter writer{buffer}; writer.startObject(); attributesFactory(writer); - m_attributesJson = std::move(stream); + m_attributesJson = std::move(buffer); } Journal& @@ -758,7 +759,7 @@ class Journal static void resetGlobalAttributes() { - std::lock_guard lock(globalLogAttributesMutex_); + std::unique_lock lock(globalLogAttributesMutex_); globalLogAttributesJson_.clear(); } @@ -766,17 +767,17 @@ class Journal static void addGlobalAttributes(TAttributesFactory&& factory) { - std::lock_guard lock(globalLogAttributesMutex_); + std::unique_lock lock(globalLogAttributesMutex_); auto isEmpty = globalLogAttributesJson_.empty(); - std::string stream{std::move(globalLogAttributesJson_)}; - detail::SimpleJsonWriter writer{stream}; + std::string buffer{std::move(globalLogAttributesJson_)}; + detail::SimpleJsonWriter writer{buffer}; if (isEmpty) { writer.startObject(); } factory(writer); - globalLogAttributesJson_ = std::move(stream); + globalLogAttributesJson_ = std::move(buffer); } }; diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index 9c485cdfa82..47874104e09 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -22,13 +22,14 @@ #include #include #include +#include #include #include namespace beast { std::string Journal::globalLogAttributesJson_; -std::mutex Journal::globalLogAttributesMutex_; +std::shared_mutex Journal::globalLogAttributesMutex_; bool Journal::m_jsonLogsEnabled = false; thread_local Journal::JsonLogContext Journal::currentJsonLogContext_{}; @@ -150,13 +151,16 @@ Journal::JsonLogContext::reset( writer().endObject(); } - if (!globalLogAttributesJson_.empty()) { - writer().writeKey("GlobalParams"); - writer().writeRaw(std::string_view{ - std::begin(globalLogAttributesJson_), - std::end(globalLogAttributesJson_)}); - writer().endObject(); + std::shared_lock lock(globalLogAttributesMutex_); + if (!globalLogAttributesJson_.empty()) + { + writer().writeKey("GlobalParams"); + writer().writeRaw(std::string_view{ + std::begin(globalLogAttributesJson_), + std::end(globalLogAttributesJson_)}); + writer().endObject(); + } } writer().writeKey("MessageParams"); diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index 7651d218e57..48db448ada6 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -231,32 +231,32 @@ TEST_CASE("Global attributes inheritable") TEST_CASE("Test JsonWriter") { { - std::string stream; - beast::detail::SimpleJsonWriter writer{stream}; + std::string buffer; + beast::detail::SimpleJsonWriter writer{buffer}; writer.writeString("\n"); CHECK(writer.finish() == "\"\\n\""); } { - std::string stream; - beast::detail::SimpleJsonWriter writer{stream}; + std::string buffer; + beast::detail::SimpleJsonWriter writer{buffer}; writer.writeString("\t"); CHECK(writer.finish() == "\"\\t\""); } { - std::string stream; - beast::detail::SimpleJsonWriter writer{stream}; + std::string buffer; + beast::detail::SimpleJsonWriter writer{buffer}; writer.writeString(std::string_view{"\0", 1}); CHECK(writer.finish() == "\"\\u0000\""); } { - std::string stream; - beast::detail::SimpleJsonWriter writer{stream}; + std::string buffer; + beast::detail::SimpleJsonWriter writer{buffer}; writer.writeString("\"\\"); CHECK(writer.finish() == "\"\\\"\\\\\""); From 4f63747f33d3a89e7032e7209d41912a4412c5c4 Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 2 Sep 2025 20:21:05 +0100 Subject: [PATCH 46/88] Polish code Signed-off-by: JCW --- src/libxrpl/beast/utility/beast_Journal.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index 47874104e09..c8d677b5fe8 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -156,9 +156,7 @@ Journal::JsonLogContext::reset( if (!globalLogAttributesJson_.empty()) { writer().writeKey("GlobalParams"); - writer().writeRaw(std::string_view{ - std::begin(globalLogAttributesJson_), - std::end(globalLogAttributesJson_)}); + writer().writeRaw(globalLogAttributesJson_); writer().endObject(); } } @@ -196,8 +194,7 @@ Journal::initMessageContext( currentJsonLogContext_.reset( location, severity, - std::string_view{ - std::begin(m_attributesJson), std::end(m_attributesJson)}); + m_attributesJson); } std::string_view From 4d0c0ca5c754a893eec2ddeecc3fa4c1378a17b9 Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 2 Sep 2025 20:24:01 +0100 Subject: [PATCH 47/88] Polish code Signed-off-by: JCW --- src/libxrpl/beast/utility/beast_Journal.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index c8d677b5fe8..25f62ec4210 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -171,7 +171,7 @@ Journal::JsonLogContext::reset( writer().writeString(location.file_name()); writer().writeKey("Line"); - writer().writeInt(static_cast(location.line())); + writer().writeUInt(location.line()); writer().writeKey("ThreadId"); writer().writeString(threadId.value); From e6c5f8338b2f2ea560f0532fcc9a12ba7b882ce0 Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 2 Sep 2025 20:32:52 +0100 Subject: [PATCH 48/88] Fix formatting Signed-off-by: JCW --- src/libxrpl/beast/utility/beast_Journal.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index 25f62ec4210..97e7c47bb7b 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -191,10 +191,7 @@ Journal::initMessageContext( std::source_location location, severities::Severity severity) const { - currentJsonLogContext_.reset( - location, - severity, - m_attributesJson); + currentJsonLogContext_.reset(location, severity, m_attributesJson); } std::string_view From 28ad89ca200db5ebdc389c223ec22589485a4473 Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 2 Sep 2025 21:34:03 +0100 Subject: [PATCH 49/88] Fix build error Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 43 +++++++++++----------------- 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 2800929785b..4e9b2bb9e23 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -128,16 +128,6 @@ class SimpleJsonWriter buffer_.append("\","sv); } std::string_view - writeInt(std::int8_t val) const - { - return pushNumber(val, buffer_); - } - std::string_view - writeInt(std::int16_t val) const - { - return pushNumber(val, buffer_); - } - std::string_view writeInt(std::int32_t val) const { return pushNumber(val, buffer_); @@ -148,21 +138,6 @@ class SimpleJsonWriter return pushNumber(val, buffer_); } std::string_view - writeUInt(std::size_t val) const - { - return pushNumber(val, buffer_); - } - std::string_view - writeUInt(std::uint8_t val) const - { - return pushNumber(val, buffer_); - } - std::string_view - writeUInt(std::uint16_t val) const - { - return pushNumber(val, buffer_); - } - std::string_view writeUInt(std::uint32_t val) const { return pushNumber(val, buffer_); @@ -919,11 +894,25 @@ setJsonValue( std::string_view sv; if constexpr (std::is_signed_v) { - sv = writer.writeInt(value); + if constexpr (sizeof(ValueType) > 4) + { + sv = writer.writeInt(static_cast(value)); + } + else + { + sv = writer.writeInt(static_cast(value)); + } } else { - sv = writer.writeUInt(value); + if constexpr (sizeof(ValueType) > 4) + { + sv = writer.writeUInt(static_cast(value)); + } + else + { + sv = writer.writeUInt(static_cast(value)); + } } if (outStream) { From 3ec7596170c60a7fa368c8dd2fd8624942fae7fc Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 2 Sep 2025 21:38:14 +0100 Subject: [PATCH 50/88] Fix build error Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 4e9b2bb9e23..48ab21c1c45 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -23,6 +23,7 @@ #include #include +#include #include #include #include From fa0cff353246e20a8281342299865de6d61c52e7 Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 2 Sep 2025 21:50:38 +0100 Subject: [PATCH 51/88] Fix build error Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 48ab21c1c45..35d41c1c246 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -23,6 +23,7 @@ #include #include +#include #include #include #include From 816089eab7206cc80bbf466c43fb8601fd6bca52 Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 2 Sep 2025 22:09:44 +0100 Subject: [PATCH 52/88] Fix error Signed-off-by: JCW --- src/tests/libxrpl/basics/log.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index 48db448ada6..46adc3592ff 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -502,14 +502,15 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogParams") logValue.as_object()["MessageParams"] .as_object()["Field2"] .get_uint64() == std::numeric_limits::max()); - CHECK( - logValue.as_object()["MessageParams"] + auto field3Val = logValue.as_object()["MessageParams"] .as_object()["Field3"] - .get_double() == 3.141593); + .get_double(); + auto difference = std::abs(field3Val - std::numbers::pi); + CHECK(difference < 1e-4); CHECK(logValue.as_object()["Message"].is_string()); CHECK( logValue.as_object()["Message"].get_string() == - std::string{"Test: 1, 18446744073709551615, 3.141593"}); + std::string{"Test: 1, 18446744073709551615, 3.141592653589793"}); } TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") From 9543ccf8e1c02631e77ee971e770ed1f3b969fd6 Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 2 Sep 2025 22:17:01 +0100 Subject: [PATCH 53/88] Set module name in json Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 1 + src/libxrpl/beast/utility/beast_Journal.cpp | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 35d41c1c246..7082528a200 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -337,6 +337,7 @@ class Journal reset( std::source_location location, severities::Severity severity, + std::string_view moduleName, std::string_view journalAttributesJson) noexcept; }; diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index 97e7c47bb7b..714fac3cd00 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -125,6 +125,7 @@ void Journal::JsonLogContext::reset( std::source_location location, severities::Severity severity, + std::string_view moduleName, std::string_view journalAttributesJson) noexcept { struct ThreadIdStringInitializer @@ -161,6 +162,8 @@ Journal::JsonLogContext::reset( } } + writer().writeKey("ModuleName"); + writer().writeString(moduleName); writer().writeKey("MessageParams"); writer().startObject(); @@ -191,7 +194,7 @@ Journal::initMessageContext( std::source_location location, severities::Severity severity) const { - currentJsonLogContext_.reset(location, severity, m_attributesJson); + currentJsonLogContext_.reset(location, severity, m_name, m_attributesJson); } std::string_view From 06c212495d1d11d0810744d4374666f87ad4920f Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 2 Sep 2025 22:17:46 +0100 Subject: [PATCH 54/88] Fix formatting Signed-off-by: JCW --- src/tests/libxrpl/basics/log.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index 46adc3592ff..9c92c6e4f3f 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -503,8 +503,8 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogParams") .as_object()["Field2"] .get_uint64() == std::numeric_limits::max()); auto field3Val = logValue.as_object()["MessageParams"] - .as_object()["Field3"] - .get_double(); + .as_object()["Field3"] + .get_double(); auto difference = std::abs(field3Val - std::numbers::pi); CHECK(difference < 1e-4); CHECK(logValue.as_object()["Message"].is_string()); From 2f6d133169b509a1dd22b51e30e666aeb24c26ed Mon Sep 17 00:00:00 2001 From: JCW Date: Wed, 3 Sep 2025 15:03:13 +0100 Subject: [PATCH 55/88] Improve test coverage Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 43 +++++--- src/tests/libxrpl/basics/log.cpp | 153 ++++++++++++++++++++++++--- 2 files changed, 172 insertions(+), 24 deletions(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 7082528a200..433bc784cad 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -638,7 +638,7 @@ class Journal operator=(Journal const& other) { if (&other == this) - return *this; + return *this; // LCOV_EXCL_LINE m_sink = other.m_sink; m_name = other.m_name; @@ -874,6 +874,13 @@ concept ToCharsFormattable = requires(T val) { } -> std::convertible_to; }; +template +concept StreamFormattable = requires(T val) { + { + std::declval() << val + } -> std::convertible_to; +}; + template void setJsonValue( @@ -941,12 +948,14 @@ setJsonValue( outStream->write(value, std::strlen(value)); } } - else if constexpr (std::is_same_v) + else if constexpr ( + std::is_same_v || + std::is_same_v) { writer.writeString(value); if (outStream) { - outStream->write(value.c_str(), value.length()); + outStream->write(value.data(), value.size()); } } else @@ -968,19 +977,26 @@ setJsonValue( } } - std::ostringstream oss; - oss.imbue(std::locale::classic()); - oss << value; + if constexpr (StreamFormattable) + { + std::ostringstream oss; + oss.imbue(std::locale::classic()); + oss << value; - auto str = oss.str(); + auto str = oss.str(); - writer.writeString(str); + writer.writeString(str); - if (outStream) - { - outStream->write( - str.c_str(), static_cast(str.size())); + if (outStream) + { + outStream->write( + str.c_str(), static_cast(str.size())); + } + + return; } + + static_assert(ToCharsFormattable || StreamFormattable); } } } // namespace detail @@ -990,7 +1006,10 @@ std::ostream& operator<<(std::ostream& os, LogParameter const& param) { if (!beast::Journal::m_jsonLogsEnabled) + { + os << param.value_; return os; + } detail::setJsonValue( beast::Journal::currentJsonLogContext_.writer(), param.name_, diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index 9c92c6e4f3f..49b070d994e 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -234,8 +234,8 @@ TEST_CASE("Test JsonWriter") std::string buffer; beast::detail::SimpleJsonWriter writer{buffer}; - writer.writeString("\n"); - CHECK(writer.finish() == "\"\\n\""); + writer.writeString("\n\r\t123\b\f123"); + CHECK(writer.finish() == "\"\\n\\r\\t123\\b\\f123\""); } { @@ -261,6 +261,76 @@ TEST_CASE("Test JsonWriter") writer.writeString("\"\\"); CHECK(writer.finish() == "\"\\\"\\\\\""); } + + { + std::string buffer; + beast::detail::SimpleJsonWriter writer{buffer}; + + writer.startArray(); + writer.writeBool(true); + writer.writeBool(false); + writer.writeNull(); + writer.endArray(); + CHECK(writer.finish() == "[true,false,null]"); + } +} + +namespace test_detail { +struct ToCharsStruct{}; + +std::to_chars_result +to_chars(char* first, char* last, ToCharsStruct) +{ + *first = '0'; + return std::to_chars_result{first + 1, std::errc{}}; +} + +struct StreamStruct +{ +}; + +std::ostream& +operator<<(std::ostream& os, StreamStruct) +{ + os << "0"; + return os; +} + +} + +TEST_CASE("Test setJsonValue") +{ + std::ostringstream stringBuf; + std::string buffer; + beast::detail::SimpleJsonWriter writer{buffer}; + writer.startObject(); + + log::detail::setJsonValue(writer, "testBool", true, &stringBuf); + log::detail::setJsonValue(writer, "testInt32", 1, &stringBuf); + log::detail::setJsonValue(writer, "testUInt32", -1, &stringBuf); + log::detail::setJsonValue(writer, "testInt64", 1, &stringBuf); + log::detail::setJsonValue(writer, "testUInt64", -1, &stringBuf); + log::detail::setJsonValue(writer, "testDouble", 1.1, &stringBuf); + log::detail::setJsonValue(writer, "testCharStar", "Char*", &stringBuf); + log::detail::setJsonValue(writer, "testStdString", "StdString", &stringBuf); + log::detail::setJsonValue(writer, "testStdStringView", "StdStringView", &stringBuf); + log::detail::setJsonValue(writer, "testToChars", {}, &stringBuf); + log::detail::setJsonValue(writer, "testStream", {}, &stringBuf); +} + +TEST_CASE("Test json logging not enabled") +{ + std::string logStream; + + MockLogs logs{logStream, beast::severities::kAll}; + + beast::Journal::disableStructuredJournal(); + beast::Journal::addGlobalAttributes( + log::attributes(log::attr("Field1", "Value1"))); + + logs.journal("Test123").debug() << "Test " << log::param(" Field1", "Value1") << log::field("Field2", "Value2"); + + CHECK(logStream.find("Test Value1") != std::string::npos); } /** @@ -322,7 +392,7 @@ class JsonLogStreamFixture beast::Journal j_; }; -TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") +TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log fields") { beast::Journal::addGlobalAttributes( log::attributes(log::attr("Field2", "Value2"))); @@ -367,7 +437,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") std::string{"true Test false"}); } -TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") +TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log levels") { { stream().str(""); @@ -460,7 +530,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogLevels") } } -TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogStream") +TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log stream") { journal().stream(beast::severities::kError) << "Test"; @@ -475,7 +545,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogStream") beast::severities::to_string(beast::severities::kError)); } -TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogParams") +TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log params") { journal().debug() << "Test: " << log::param("Field1", 1) << ", " << log::param( @@ -513,7 +583,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogParams") std::string{"Test: 1, 18446744073709551615, 3.141592653589793"}); } -TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") +TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log fields") { journal().debug() << "Test" << log::field("Field1", 1) << log::field( @@ -546,7 +616,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJsonLogFields") CHECK(logValue.as_object()["Message"].get_string() == "Test"); } -TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributes") +TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test journal attributes") { beast::Journal j{ journal(), @@ -574,7 +644,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributes") .get_int64() == 2); } -TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributesInheritable") +TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test journal attributes inheritable") { beast::Journal j{ journal(), @@ -610,9 +680,68 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "TestJournalAttributesInheritable") .get_int64() == 2); } +TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test copying journal") +{ + { + beast::Journal j{ + journal(), + log::attributes(log::attr("Field1", "Value1"), log::attr("Field2", 2))}; + beast::Journal j2{j}; + + j2.debug() << "Test"; + + boost::system::error_code ec; + auto logValue = boost::json::parse(stream().str(), ec); + CHECK(ec == boost::system::errc::success); + + CHECK(logValue.as_object()["JournalParams"] + .as_object()["Field1"] + .is_string()); + CHECK( + logValue.as_object()["JournalParams"] + .as_object()["Field1"] + .get_string() == std::string{"Value1"}); + CHECK(logValue.as_object()["JournalParams"] + .as_object()["Field2"] + .is_number()); + CHECK( + logValue.as_object()["JournalParams"] + .as_object()["Field2"] + .get_int64() == 2); + } + { + stream().str(""); + beast::Journal j{ + journal().sink()}; + beast::Journal j2{j, + log::attributes(log::attr("Field1", "Value1"), log::attr("Field2", 2))}; + + j2.debug() << "Test"; + + boost::system::error_code ec; + auto logValue = boost::json::parse(stream().str(), ec); + CHECK(ec == boost::system::errc::success); + + CHECK(logValue.as_object()["JournalParams"] + .as_object()["Field1"] + .is_string()); + CHECK( + logValue.as_object()["JournalParams"] + .as_object()["Field1"] + .get_string() == std::string{"Value1"}); + CHECK(logValue.as_object()["JournalParams"] + .as_object()["Field2"] + .is_number()); + CHECK( + logValue.as_object()["JournalParams"] + .as_object()["Field2"] + .get_int64() == 2); + } +} + TEST_CASE_FIXTURE( JsonLogStreamFixture, - "TestJournalAttributesInheritableAfterMoving") + "Test journal attributes inheritable after moving") { beast::Journal j{ journal(), @@ -651,7 +780,7 @@ TEST_CASE_FIXTURE( TEST_CASE_FIXTURE( JsonLogStreamFixture, - "TestJournalAttributesInheritableAfterCopyAssignment") + "Test journal attributes inheritable after copy assignment") { beast::Journal j{ std::move(journal()), @@ -685,7 +814,7 @@ TEST_CASE_FIXTURE( TEST_CASE_FIXTURE( JsonLogStreamFixture, - "TestJournalAttributesInheritableAfterMoveAssignment") + "Test journal attributes inheritable after move assignment") { beast::Journal j{ journal(), From 5e16b3df62d5ddacb4a0050d071d6f5b86d3ad6b Mon Sep 17 00:00:00 2001 From: JCW Date: Wed, 3 Sep 2025 15:05:16 +0100 Subject: [PATCH 56/88] Fix formatting Signed-off-by: JCW --- src/tests/libxrpl/basics/log.cpp | 43 +++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index 49b070d994e..7f8c1f26a1f 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -276,7 +276,9 @@ TEST_CASE("Test JsonWriter") } namespace test_detail { -struct ToCharsStruct{}; +struct ToCharsStruct +{ +}; std::to_chars_result to_chars(char* first, char* last, ToCharsStruct) @@ -296,7 +298,7 @@ operator<<(std::ostream& os, StreamStruct) return os; } -} +} // namespace test_detail TEST_CASE("Test setJsonValue") { @@ -307,15 +309,22 @@ TEST_CASE("Test setJsonValue") log::detail::setJsonValue(writer, "testBool", true, &stringBuf); log::detail::setJsonValue(writer, "testInt32", 1, &stringBuf); - log::detail::setJsonValue(writer, "testUInt32", -1, &stringBuf); + log::detail::setJsonValue( + writer, "testUInt32", -1, &stringBuf); log::detail::setJsonValue(writer, "testInt64", 1, &stringBuf); - log::detail::setJsonValue(writer, "testUInt64", -1, &stringBuf); + log::detail::setJsonValue( + writer, "testUInt64", -1, &stringBuf); log::detail::setJsonValue(writer, "testDouble", 1.1, &stringBuf); - log::detail::setJsonValue(writer, "testCharStar", "Char*", &stringBuf); - log::detail::setJsonValue(writer, "testStdString", "StdString", &stringBuf); - log::detail::setJsonValue(writer, "testStdStringView", "StdStringView", &stringBuf); - log::detail::setJsonValue(writer, "testToChars", {}, &stringBuf); - log::detail::setJsonValue(writer, "testStream", {}, &stringBuf); + log::detail::setJsonValue( + writer, "testCharStar", "Char*", &stringBuf); + log::detail::setJsonValue( + writer, "testStdString", "StdString", &stringBuf); + log::detail::setJsonValue( + writer, "testStdStringView", "StdStringView", &stringBuf); + log::detail::setJsonValue( + writer, "testToChars", {}, &stringBuf); + log::detail::setJsonValue( + writer, "testStream", {}, &stringBuf); } TEST_CASE("Test json logging not enabled") @@ -328,7 +337,9 @@ TEST_CASE("Test json logging not enabled") beast::Journal::addGlobalAttributes( log::attributes(log::attr("Field1", "Value1"))); - logs.journal("Test123").debug() << "Test " << log::param(" Field1", "Value1") << log::field("Field2", "Value2"); + logs.journal("Test123").debug() + << "Test " << log::param(" Field1", "Value1") + << log::field("Field2", "Value2"); CHECK(logStream.find("Test Value1") != std::string::npos); } @@ -685,7 +696,8 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test copying journal") { beast::Journal j{ journal(), - log::attributes(log::attr("Field1", "Value1"), log::attr("Field2", 2))}; + log::attributes( + log::attr("Field1", "Value1"), log::attr("Field2", 2))}; beast::Journal j2{j}; j2.debug() << "Test"; @@ -711,10 +723,11 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test copying journal") } { stream().str(""); - beast::Journal j{ - journal().sink()}; - beast::Journal j2{j, - log::attributes(log::attr("Field1", "Value1"), log::attr("Field2", 2))}; + beast::Journal j{journal().sink()}; + beast::Journal j2{ + j, + log::attributes( + log::attr("Field1", "Value1"), log::attr("Field2", 2))}; j2.debug() << "Test"; From 44aa394e1e91ed1834fc7f9d7a2f68735033e34e Mon Sep 17 00:00:00 2001 From: JCW Date: Wed, 3 Sep 2025 15:05:23 +0100 Subject: [PATCH 57/88] Fix formatting Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 433bc784cad..66df05379a6 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -638,7 +638,7 @@ class Journal operator=(Journal const& other) { if (&other == this) - return *this; // LCOV_EXCL_LINE + return *this; // LCOV_EXCL_LINE m_sink = other.m_sink; m_name = other.m_name; @@ -996,7 +996,8 @@ setJsonValue( return; } - static_assert(ToCharsFormattable || StreamFormattable); + static_assert( + ToCharsFormattable || StreamFormattable); } } } // namespace detail From 79e8c6a1589e7702dcdeb156f845d79cc9e27f2c Mon Sep 17 00:00:00 2001 From: JCW Date: Wed, 3 Sep 2025 16:12:24 +0100 Subject: [PATCH 58/88] Add additional check Signed-off-by: JCW --- src/tests/libxrpl/basics/log.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index 7f8c1f26a1f..fb2dcda5fdb 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -308,12 +308,12 @@ TEST_CASE("Test setJsonValue") writer.startObject(); log::detail::setJsonValue(writer, "testBool", true, &stringBuf); - log::detail::setJsonValue(writer, "testInt32", 1, &stringBuf); + log::detail::setJsonValue(writer, "testInt32", -1, &stringBuf); log::detail::setJsonValue( - writer, "testUInt32", -1, &stringBuf); - log::detail::setJsonValue(writer, "testInt64", 1, &stringBuf); + writer, "testUInt32", 1, &stringBuf); + log::detail::setJsonValue(writer, "testInt64", -1, &stringBuf); log::detail::setJsonValue( - writer, "testUInt64", -1, &stringBuf); + writer, "testUInt64", 1, &stringBuf); log::detail::setJsonValue(writer, "testDouble", 1.1, &stringBuf); log::detail::setJsonValue( writer, "testCharStar", "Char*", &stringBuf); @@ -325,6 +325,9 @@ TEST_CASE("Test setJsonValue") writer, "testToChars", {}, &stringBuf); log::detail::setJsonValue( writer, "testStream", {}, &stringBuf); + writer.endObject(); + + CHECK(writer.finish() == R"AAA({"testBool":true,"testInt32":-1,"testUInt32":1,"testInt64":-1,"testUInt64":1,"testDouble":1.1,"testCharStar":"Char*","testStdString":"StdString","testStdStringView":"StdStringView","testToChars":"0","testStream":"0"})AAA"); } TEST_CASE("Test json logging not enabled") From 0223443452ee176095c8c9e3337928b5720fbdce Mon Sep 17 00:00:00 2001 From: JCW Date: Wed, 3 Sep 2025 16:36:27 +0100 Subject: [PATCH 59/88] Hardcode the logstyle as json Signed-off-by: JCW --- src/xrpld/core/detail/Config.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/xrpld/core/detail/Config.cpp b/src/xrpld/core/detail/Config.cpp index 7eb3a68a46d..7a5301389e2 100644 --- a/src/xrpld/core/detail/Config.cpp +++ b/src/xrpld/core/detail/Config.cpp @@ -693,6 +693,8 @@ Config::loadFromString(std::string const& fileContents) if (getSingleSection(secConfig, SECTION_LOG_STYLE, strTemp, j_)) LOG_STYLE = LogStyle::fromString(strTemp); + LOG_STYLE = LogStyle::Json; + if (getSingleSection(secConfig, SECTION_SWEEP_INTERVAL, strTemp, j_)) { SWEEP_INTERVAL = beast::lexicalCastThrow(strTemp); From bb787e39956f1e5965a5a07848aa353056e6a343 Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 4 Sep 2025 16:03:52 +0100 Subject: [PATCH 60/88] Optimisation Signed-off-by: JCW --- include/xrpl/basics/Log.h | 41 +++---- include/xrpl/beast/utility/Journal.h | 25 ++-- src/libxrpl/basics/Log.cpp | 84 ++++++++++--- src/libxrpl/beast/utility/beast_Journal.cpp | 123 ++++++++++++++++++-- src/xrpld/app/main/Main.cpp | 1 + 5 files changed, 215 insertions(+), 59 deletions(-) diff --git a/include/xrpl/basics/Log.h b/include/xrpl/basics/Log.h index 95d4150e2f1..617c2ebbeda 100644 --- a/include/xrpl/basics/Log.h +++ b/include/xrpl/basics/Log.h @@ -31,6 +31,9 @@ #include #include #include +#include +#include +#include namespace ripple { @@ -129,33 +132,13 @@ class Logs Does nothing if there is no associated system file. */ void - write(char const* text); + write(std::string_view str); - /** write to the log file and append an end of line marker. - Does nothing if there is no associated system file. - */ - void - writeln(char const* text); - - /** Write to the log file using std::string. */ - /** @{ */ - void - write(std::string const& str) - { - write(str.c_str()); - } - - void - writeln(std::string const& str) - { - writeln(str.c_str()); - } /** @} */ private: std::unique_ptr m_stream; boost::filesystem::path m_path; - std::mutex mutable fileMutex_; }; std::mutex mutable sinkSetMutex_; @@ -167,6 +150,14 @@ class Logs beast::severities::Severity thresh_; File file_; bool silent_ = false; + + // Batching members + mutable std::mutex batchMutex_; + static constexpr size_t BATCH_BUFFER_SIZE = 64 * 1024; // 64KB buffer + std::array batchBuffer_; + std::span writeBuffer_; // Points to available write space + std::span readBuffer_; // Points to data ready to flush + std::chrono::steady_clock::time_point lastFlush_ = std::chrono::steady_clock::now(); public: Logs(beast::severities::Severity level); @@ -175,7 +166,7 @@ class Logs Logs& operator=(Logs const&) = delete; - virtual ~Logs() = default; + virtual ~Logs(); // Need to flush on destruction bool open(boost::filesystem::path const& pathToLogFile); @@ -218,6 +209,9 @@ class Logs std::string rotate(); + + void + flushBatch(); /** * Set flag to write logs to stderr (false) or not (true). @@ -261,6 +255,9 @@ class Logs // If the message exceeds this length it will be truncated with elipses. maximumMessageCharacters = 12 * 1024 }; + + void + flushBatchUnsafe(); }; // Wraps a Journal::Stream to skip evaluation of diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 66df05379a6..41bf4281303 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -22,10 +22,11 @@ #include +#include #include #include +#include #include -#include #include #include #include @@ -347,8 +348,8 @@ class Journal std::string m_name; std::string m_attributesJson; - static std::string globalLogAttributesJson_; - static std::shared_mutex globalLogAttributesMutex_; + static std::atomic> globalLogAttributesJson_; + static std::size_t m_filePathOffset_; static bool m_jsonLogsEnabled; static thread_local JsonLogContext currentJsonLogContext_; @@ -367,6 +368,9 @@ class Journal public: //-------------------------------------------------------------------------- + static void + setRootPath(std::string_view fullPath, std::string_view relativePath); + static void enableStructuredJournal(); @@ -628,6 +632,7 @@ class Journal : m_name(name), m_sink(&sink) { std::string buffer; + buffer.reserve(128); detail::SimpleJsonWriter writer{buffer}; writer.startObject(); attributesFactory(writer); @@ -737,25 +742,25 @@ class Journal static void resetGlobalAttributes() { - std::unique_lock lock(globalLogAttributesMutex_); - globalLogAttributesJson_.clear(); + globalLogAttributesJson_.store(std::make_shared()); } template static void addGlobalAttributes(TAttributesFactory&& factory) { - std::unique_lock lock(globalLogAttributesMutex_); - - auto isEmpty = globalLogAttributesJson_.empty(); - std::string buffer{std::move(globalLogAttributesJson_)}; + auto current = globalLogAttributesJson_.load(); + std::string buffer = current ? *current : std::string{}; + buffer.reserve(128); + auto isEmpty = buffer.empty(); detail::SimpleJsonWriter writer{buffer}; if (isEmpty) { writer.startObject(); } factory(writer); - globalLogAttributesJson_ = std::move(buffer); + + globalLogAttributesJson_.store(std::make_shared(std::move(buffer))); } }; diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index 197b4e16e33..0e0e2fb6876 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -38,6 +38,10 @@ namespace ripple { +namespace { + constexpr auto FLUSH_INTERVAL = std::chrono::milliseconds(10); // Max delay before flush +} + Logs::Sink::Sink( std::string const& partition, beast::severities::Severity thresh, @@ -86,7 +90,6 @@ Logs::File::open(boost::filesystem::path const& path) if (stream->good()) { - std::lock_guard lock(fileMutex_); m_path = path; m_stream = std::move(stream); @@ -111,33 +114,28 @@ Logs::File::closeAndReopen() void Logs::File::close() { - std::lock_guard lock(fileMutex_); m_stream = nullptr; } void -Logs::File::write(char const* text) +Logs::File::write(std::string_view text) { - std::lock_guard lock(fileMutex_); if (m_stream != nullptr) - (*m_stream) << text; -} - -void -Logs::File::writeln(char const* text) -{ - std::lock_guard lock(fileMutex_); - if (m_stream != nullptr) - { - (*m_stream) << text << '\n'; - } + m_stream->write(text.data(), text.size()); } //------------------------------------------------------------------------------ Logs::Logs(beast::severities::Severity thresh) : thresh_(thresh) // default severity + , writeBuffer_(batchBuffer_) // Initially, entire buffer is available for writing + , readBuffer_(batchBuffer_.data(), 0) // No data ready to flush initially +{ +} + +Logs::~Logs() { + flushBatch(); // Ensure all logs are written on shutdown } bool @@ -195,17 +193,71 @@ Logs::write( { std::string s; format(s, text, level, partition); - file_.writeln(s); + + // Console output still immediate for responsiveness if (!silent_) std::cerr << s << '\n'; + + // Add to batch buffer for file output + { + std::lock_guard lock(batchMutex_); + + size_t logSize = s.size() + 1; // +1 for newline + + // If log won't fit in current write buffer, flush first + if (logSize > writeBuffer_.size()) { + flushBatchUnsafe(); + } + + // Copy log into write buffer + std::copy(s.begin(), s.end(), writeBuffer_.begin()); + writeBuffer_[s.size()] = '\n'; + + // Update spans: expand read buffer, shrink write buffer + size_t totalUsed = readBuffer_.size() + logSize; + readBuffer_ = std::span(batchBuffer_.data(), totalUsed); + writeBuffer_ = std::span(batchBuffer_.data() + totalUsed, + batchBuffer_.size() - totalUsed); + + auto now = std::chrono::steady_clock::now(); + bool shouldFlush = (now - lastFlush_) >= FLUSH_INTERVAL; + + if (shouldFlush) { + flushBatchUnsafe(); + lastFlush_ = now; + } + } + // VFALCO TODO Fix console output // if (console) // out_.write_console(s); } +void +Logs::flushBatch() +{ + std::lock_guard lock(batchMutex_); + flushBatchUnsafe(); +} + +void +Logs::flushBatchUnsafe() +{ + if (readBuffer_.empty()) + return; + + // Write the read buffer contents to file in one system call + file_.write(std::string_view{readBuffer_.data(), readBuffer_.size()}); + + // Reset spans: entire buffer available for writing, nothing to read + writeBuffer_ = std::span(batchBuffer_); + readBuffer_ = std::span(batchBuffer_.data(), 0); +} + std::string Logs::rotate() { + flushBatch(); // Flush pending logs before rotating bool const wasOpened = file_.closeAndReopen(); if (wasOpened) return "The log file was closed and reopened."; diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index 714fac3cd00..b318e2e00ca 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -22,14 +22,91 @@ #include #include #include -#include #include #include +#include namespace beast { -std::string Journal::globalLogAttributesJson_; -std::shared_mutex Journal::globalLogAttributesMutex_; +namespace { + +// Fast timestamp to ISO string conversion +// Returns string like "2024-01-15T10:30:45.123Z" +std::string_view +fastTimestampToString(std::int64_t milliseconds_since_epoch) +{ + thread_local char buffer[64]; // "2024-01-15T10:30:45.123Z" + + // Precomputed lookup table for 2-digit numbers 00-99 + static constexpr char digits[200] = { + '0','0','0','1','0','2','0','3','0','4','0','5','0','6','0','7','0','8','0','9', + '1','0','1','1','1','2','1','3','1','4','1','5','1','6','1','7','1','8','1','9', + '2','0','2','1','2','2','2','3','2','4','2','5','2','6','2','7','2','8','2','9', + '3','0','3','1','3','2','3','3','3','4','3','5','3','6','3','7','3','8','3','9', + '4','0','4','1','4','2','4','3','4','4','4','5','4','6','4','7','4','8','4','9', + '5','0','5','1','5','2','5','3','5','4','5','5','5','6','5','7','5','8','5','9', + '6','0','6','1','6','2','6','3','6','4','6','5','6','6','6','7','6','8','6','9', + '7','0','7','1','7','2','7','3','7','4','7','5','7','6','7','7','7','8','7','9', + '8','0','8','1','8','2','8','3','8','4','8','5','8','6','8','7','8','8','8','9', + '9','0','9','1','9','2','9','3','9','4','9','5','9','6','9','7','9','8','9','9' + }; + + constexpr std::int64_t UNIX_EPOCH_DAYS = 719468; // Days from year 0 to 1970-01-01 + + std::int64_t seconds = milliseconds_since_epoch / 1000; + int ms = milliseconds_since_epoch % 1000; + std::int64_t days = seconds / 86400 + UNIX_EPOCH_DAYS; + int sec_of_day = seconds % 86400; + + // Calculate year, month, day from days using Gregorian calendar algorithm + int era = (days >= 0 ? days : days - 146096) / 146097; + int doe = days - era * 146097; + int yoe = (doe - doe/1460 + doe/36524 - doe/146096) / 365; + int year = yoe + era * 400; + int doy = doe - (365*yoe + yoe/4 - yoe/100); + int mp = (5*doy + 2)/153; + int day = doy - (153*mp+2)/5 + 1; + int month = mp + (mp < 10 ? 3 : -9); + year += (month <= 2); + + // Calculate hour, minute, second + int hour = sec_of_day / 3600; + int min = (sec_of_day % 3600) / 60; + int sec = sec_of_day % 60; + + // Format: "2024-01-15T10:30:45.123Z" + buffer[0] = '0' + year / 1000; + buffer[1] = '0' + (year / 100) % 10; + buffer[2] = '0' + (year / 10) % 10; + buffer[3] = '0' + year % 10; + buffer[4] = '-'; + buffer[5] = digits[month * 2]; + buffer[6] = digits[month * 2 + 1]; + buffer[7] = '-'; + buffer[8] = digits[day * 2]; + buffer[9] = digits[day * 2 + 1]; + buffer[10] = 'T'; + buffer[11] = digits[hour * 2]; + buffer[12] = digits[hour * 2 + 1]; + buffer[13] = ':'; + buffer[14] = digits[min * 2]; + buffer[15] = digits[min * 2 + 1]; + buffer[16] = ':'; + buffer[17] = digits[sec * 2]; + buffer[18] = digits[sec * 2 + 1]; + buffer[19] = '.'; + buffer[20] = '0' + ms / 100; + buffer[21] = '0' + (ms / 10) % 10; + buffer[22] = '0' + ms % 10; + buffer[23] = 'Z'; + + return {buffer, 24}; +} + +} // anonymous namespace + +std::atomic> Journal::globalLogAttributesJson_; +std::size_t Journal::m_filePathOffset_ = 0; bool Journal::m_jsonLogsEnabled = false; thread_local Journal::JsonLogContext Journal::currentJsonLogContext_{}; @@ -153,25 +230,29 @@ Journal::JsonLogContext::reset( } { - std::shared_lock lock(globalLogAttributesMutex_); - if (!globalLogAttributesJson_.empty()) + auto globalAttrs = globalLogAttributesJson_.load(); + if (globalAttrs && !globalAttrs->empty()) { writer().writeKey("GlobalParams"); - writer().writeRaw(globalLogAttributesJson_); + writer().writeRaw(*globalAttrs); writer().endObject(); } } writer().writeKey("ModuleName"); writer().writeString(moduleName); - writer().writeKey("MessageParams"); + writer().writeKey("Metadata"); writer().startObject(); writer().writeKey("Function"); writer().writeString(location.function_name()); writer().writeKey("File"); - writer().writeString(location.file_name()); + std::string_view fileName = location.file_name(); + std::string_view trimmedFileName = (m_filePathOffset_ < fileName.size()) + ? fileName.substr(m_filePathOffset_) + : fileName; + writer().writeString(trimmedFileName); writer().writeKey("Line"); writer().writeUInt(location.line()); @@ -183,10 +264,17 @@ Journal::JsonLogContext::reset( writer().writeKey("Level"); writer().writeString(severityStr); + auto nowMs = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); + writer().writeKey("Timestamp"); + writer().writeInt(nowMs); writer().writeKey("Time"); - writer().writeInt(std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count()); + writer().writeString(fastTimestampToString(nowMs)); + + writer().endObject(); + + writer().writeKey("MessageParams"); + writer().startObject(); } void @@ -216,6 +304,19 @@ Journal::formatLog(std::string&& message) return writer.finish(); } +void +Journal::setRootPath(std::string_view fullPath, std::string_view relativePath) +{ + if (relativePath.size() <= fullPath.size() && + fullPath.ends_with(relativePath)) + { + m_filePathOffset_ = fullPath.size() - relativePath.size(); + } + else + { + m_filePathOffset_ = 0; + } +} void Journal::enableStructuredJournal() diff --git a/src/xrpld/app/main/Main.cpp b/src/xrpld/app/main/Main.cpp index 1531f8c3f09..3bad24de440 100644 --- a/src/xrpld/app/main/Main.cpp +++ b/src/xrpld/app/main/Main.cpp @@ -796,6 +796,7 @@ run(int argc, char** argv) if (config->LOG_STYLE == LogStyle::Json) { + beast::Journal::setRootPath(std::source_location::current().file_name(), "src/xrpld/app/main/Main.cpp"); beast::Journal::enableStructuredJournal(); beast::Journal::addGlobalAttributes(log::attributes( log::attr("Application", "rippled"), From bd91ec724246bedb3dde1a3eac2af136245c07f4 Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 4 Sep 2025 16:31:39 +0100 Subject: [PATCH 61/88] Optimise Signed-off-by: JCW --- include/xrpl/basics/Log.h | 15 ++-- include/xrpl/beast/utility/Journal.h | 28 ++++--- src/libxrpl/basics/Log.cpp | 38 +++++---- src/libxrpl/beast/utility/beast_Journal.cpp | 86 ++++++++++----------- src/tests/libxrpl/basics/log.cpp | 54 ++++++------- src/xrpld/app/main/Application.cpp | 4 - src/xrpld/app/main/Main.cpp | 4 - 7 files changed, 107 insertions(+), 122 deletions(-) diff --git a/include/xrpl/basics/Log.h b/include/xrpl/basics/Log.h index 617c2ebbeda..2bf6e6a76d7 100644 --- a/include/xrpl/basics/Log.h +++ b/include/xrpl/basics/Log.h @@ -26,14 +26,14 @@ #include #include +#include +#include #include #include #include #include -#include -#include -#include #include +#include namespace ripple { @@ -150,14 +150,15 @@ class Logs beast::severities::Severity thresh_; File file_; bool silent_ = false; - + // Batching members mutable std::mutex batchMutex_; static constexpr size_t BATCH_BUFFER_SIZE = 64 * 1024; // 64KB buffer std::array batchBuffer_; std::span writeBuffer_; // Points to available write space std::span readBuffer_; // Points to data ready to flush - std::chrono::steady_clock::time_point lastFlush_ = std::chrono::steady_clock::now(); + std::chrono::steady_clock::time_point lastFlush_ = + std::chrono::steady_clock::now(); public: Logs(beast::severities::Severity level); @@ -209,7 +210,7 @@ class Logs std::string rotate(); - + void flushBatch(); @@ -255,7 +256,7 @@ class Logs // If the message exceeds this length it will be truncated with elipses. maximumMessageCharacters = 12 * 1024 }; - + void flushBatchUnsafe(); }; diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 41bf4281303..0e02336033a 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -101,7 +102,8 @@ class SimpleJsonWriter endObject() const { using namespace std::string_view_literals; - buffer_.pop_back(); + if (buffer_.back() == ',') + buffer_.pop_back(); buffer_.append("},"sv); } void @@ -119,7 +121,8 @@ class SimpleJsonWriter endArray() const { using namespace std::string_view_literals; - buffer_.pop_back(); + if (buffer_.back() == ',') + buffer_.pop_back(); buffer_.append("],"sv); } void @@ -348,8 +351,8 @@ class Journal std::string m_name; std::string m_attributesJson; - static std::atomic> globalLogAttributesJson_; - static std::size_t m_filePathOffset_; + static std::string globalLogAttributesJson_; + static std::shared_mutex globalLogAttributesMutex_; static bool m_jsonLogsEnabled; static thread_local JsonLogContext currentJsonLogContext_; @@ -368,9 +371,6 @@ class Journal public: //-------------------------------------------------------------------------- - static void - setRootPath(std::string_view fullPath, std::string_view relativePath); - static void enableStructuredJournal(); @@ -742,25 +742,23 @@ class Journal static void resetGlobalAttributes() { - globalLogAttributesJson_.store(std::make_shared()); + std::unique_lock lock(globalLogAttributesMutex_); + globalLogAttributesJson_.clear(); } template static void addGlobalAttributes(TAttributesFactory&& factory) { - auto current = globalLogAttributesJson_.load(); - std::string buffer = current ? *current : std::string{}; - buffer.reserve(128); - auto isEmpty = buffer.empty(); - detail::SimpleJsonWriter writer{buffer}; + std::unique_lock lock(globalLogAttributesMutex_); + globalLogAttributesJson_.reserve(128); + auto isEmpty = globalLogAttributesJson_.empty(); + detail::SimpleJsonWriter writer{globalLogAttributesJson_}; if (isEmpty) { writer.startObject(); } factory(writer); - - globalLogAttributesJson_.store(std::make_shared(std::move(buffer))); } }; diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index 0e0e2fb6876..996c8bf3f11 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -39,7 +39,8 @@ namespace ripple { namespace { - constexpr auto FLUSH_INTERVAL = std::chrono::milliseconds(10); // Max delay before flush +constexpr auto FLUSH_INTERVAL = + std::chrono::milliseconds(10); // Max delay before flush } Logs::Sink::Sink( @@ -128,7 +129,8 @@ Logs::File::write(std::string_view text) Logs::Logs(beast::severities::Severity thresh) : thresh_(thresh) // default severity - , writeBuffer_(batchBuffer_) // Initially, entire buffer is available for writing + , writeBuffer_( + batchBuffer_) // Initially, entire buffer is available for writing , readBuffer_(batchBuffer_.data(), 0) // No data ready to flush initially { } @@ -193,41 +195,43 @@ Logs::write( { std::string s; format(s, text, level, partition); - + // Console output still immediate for responsiveness if (!silent_) std::cerr << s << '\n'; - + // Add to batch buffer for file output { std::lock_guard lock(batchMutex_); - + size_t logSize = s.size() + 1; // +1 for newline - + // If log won't fit in current write buffer, flush first - if (logSize > writeBuffer_.size()) { + if (logSize > writeBuffer_.size()) + { flushBatchUnsafe(); } - + // Copy log into write buffer std::copy(s.begin(), s.end(), writeBuffer_.begin()); writeBuffer_[s.size()] = '\n'; - + // Update spans: expand read buffer, shrink write buffer size_t totalUsed = readBuffer_.size() + logSize; readBuffer_ = std::span(batchBuffer_.data(), totalUsed); - writeBuffer_ = std::span(batchBuffer_.data() + totalUsed, - batchBuffer_.size() - totalUsed); - + writeBuffer_ = std::span( + batchBuffer_.data() + totalUsed, batchBuffer_.size() - totalUsed); + auto now = std::chrono::steady_clock::now(); bool shouldFlush = (now - lastFlush_) >= FLUSH_INTERVAL; - - if (shouldFlush) { + + if (shouldFlush) + { flushBatchUnsafe(); lastFlush_ = now; } } - + // VFALCO TODO Fix console output // if (console) // out_.write_console(s); @@ -245,10 +249,10 @@ Logs::flushBatchUnsafe() { if (readBuffer_.empty()) return; - + // Write the read buffer contents to file in one system call file_.write(std::string_view{readBuffer_.data(), readBuffer_.size()}); - + // Reset spans: entire buffer available for writing, nothing to read writeBuffer_ = std::span(batchBuffer_); readBuffer_ = std::span(batchBuffer_.data(), 0); diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index b318e2e00ca..88ada9664bf 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -19,12 +19,12 @@ #include +#include #include #include #include #include #include -#include namespace beast { @@ -35,45 +35,50 @@ namespace { std::string_view fastTimestampToString(std::int64_t milliseconds_since_epoch) { - thread_local char buffer[64]; // "2024-01-15T10:30:45.123Z" - + thread_local char buffer[64]; // "2024-01-15T10:30:45.123Z" + // Precomputed lookup table for 2-digit numbers 00-99 static constexpr char digits[200] = { - '0','0','0','1','0','2','0','3','0','4','0','5','0','6','0','7','0','8','0','9', - '1','0','1','1','1','2','1','3','1','4','1','5','1','6','1','7','1','8','1','9', - '2','0','2','1','2','2','2','3','2','4','2','5','2','6','2','7','2','8','2','9', - '3','0','3','1','3','2','3','3','3','4','3','5','3','6','3','7','3','8','3','9', - '4','0','4','1','4','2','4','3','4','4','4','5','4','6','4','7','4','8','4','9', - '5','0','5','1','5','2','5','3','5','4','5','5','5','6','5','7','5','8','5','9', - '6','0','6','1','6','2','6','3','6','4','6','5','6','6','6','7','6','8','6','9', - '7','0','7','1','7','2','7','3','7','4','7','5','7','6','7','7','7','8','7','9', - '8','0','8','1','8','2','8','3','8','4','8','5','8','6','8','7','8','8','8','9', - '9','0','9','1','9','2','9','3','9','4','9','5','9','6','9','7','9','8','9','9' - }; - - constexpr std::int64_t UNIX_EPOCH_DAYS = 719468; // Days from year 0 to 1970-01-01 - + '0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', '0', '6', + '0', '7', '0', '8', '0', '9', '1', '0', '1', '1', '1', '2', '1', '3', + '1', '4', '1', '5', '1', '6', '1', '7', '1', '8', '1', '9', '2', '0', + '2', '1', '2', '2', '2', '3', '2', '4', '2', '5', '2', '6', '2', '7', + '2', '8', '2', '9', '3', '0', '3', '1', '3', '2', '3', '3', '3', '4', + '3', '5', '3', '6', '3', '7', '3', '8', '3', '9', '4', '0', '4', '1', + '4', '2', '4', '3', '4', '4', '4', '5', '4', '6', '4', '7', '4', '8', + '4', '9', '5', '0', '5', '1', '5', '2', '5', '3', '5', '4', '5', '5', + '5', '6', '5', '7', '5', '8', '5', '9', '6', '0', '6', '1', '6', '2', + '6', '3', '6', '4', '6', '5', '6', '6', '6', '7', '6', '8', '6', '9', + '7', '0', '7', '1', '7', '2', '7', '3', '7', '4', '7', '5', '7', '6', + '7', '7', '7', '8', '7', '9', '8', '0', '8', '1', '8', '2', '8', '3', + '8', '4', '8', '5', '8', '6', '8', '7', '8', '8', '8', '9', '9', '0', + '9', '1', '9', '2', '9', '3', '9', '4', '9', '5', '9', '6', '9', '7', + '9', '8', '9', '9'}; + + constexpr std::int64_t UNIX_EPOCH_DAYS = + 719468; // Days from year 0 to 1970-01-01 + std::int64_t seconds = milliseconds_since_epoch / 1000; int ms = milliseconds_since_epoch % 1000; std::int64_t days = seconds / 86400 + UNIX_EPOCH_DAYS; int sec_of_day = seconds % 86400; - + // Calculate year, month, day from days using Gregorian calendar algorithm int era = (days >= 0 ? days : days - 146096) / 146097; int doe = days - era * 146097; - int yoe = (doe - doe/1460 + doe/36524 - doe/146096) / 365; + int yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; int year = yoe + era * 400; - int doy = doe - (365*yoe + yoe/4 - yoe/100); - int mp = (5*doy + 2)/153; - int day = doy - (153*mp+2)/5 + 1; + int doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + int mp = (5 * doy + 2) / 153; + int day = doy - (153 * mp + 2) / 5 + 1; int month = mp + (mp < 10 ? 3 : -9); year += (month <= 2); - + // Calculate hour, minute, second int hour = sec_of_day / 3600; int min = (sec_of_day % 3600) / 60; int sec = sec_of_day % 60; - + // Format: "2024-01-15T10:30:45.123Z" buffer[0] = '0' + year / 1000; buffer[1] = '0' + (year / 100) % 10; @@ -99,14 +104,14 @@ fastTimestampToString(std::int64_t milliseconds_since_epoch) buffer[21] = '0' + (ms / 10) % 10; buffer[22] = '0' + ms % 10; buffer[23] = 'Z'; - + return {buffer, 24}; } } // anonymous namespace -std::atomic> Journal::globalLogAttributesJson_; -std::size_t Journal::m_filePathOffset_ = 0; +std::string Journal::globalLogAttributesJson_; +std::shared_mutex Journal::globalLogAttributesMutex_; bool Journal::m_jsonLogsEnabled = false; thread_local Journal::JsonLogContext Journal::currentJsonLogContext_{}; @@ -230,11 +235,11 @@ Journal::JsonLogContext::reset( } { - auto globalAttrs = globalLogAttributesJson_.load(); - if (globalAttrs && !globalAttrs->empty()) + std::shared_lock lock(globalLogAttributesMutex_); + if (!globalLogAttributesJson_.empty()) { writer().writeKey("GlobalParams"); - writer().writeRaw(*globalAttrs); + writer().writeRaw(globalLogAttributesJson_); writer().endObject(); } } @@ -249,8 +254,9 @@ Journal::JsonLogContext::reset( writer().writeKey("File"); std::string_view fileName = location.file_name(); - std::string_view trimmedFileName = (m_filePathOffset_ < fileName.size()) - ? fileName.substr(m_filePathOffset_) + constexpr size_t KEEP_CHARS = 10; + std::string_view trimmedFileName = (fileName.size() > KEEP_CHARS) + ? fileName.substr(fileName.size() - KEEP_CHARS) : fileName; writer().writeString(trimmedFileName); @@ -265,7 +271,8 @@ Journal::JsonLogContext::reset( writer().writeString(severityStr); auto nowMs = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()).count(); + std::chrono::system_clock::now().time_since_epoch()) + .count(); writer().writeKey("Timestamp"); writer().writeInt(nowMs); writer().writeKey("Time"); @@ -304,19 +311,6 @@ Journal::formatLog(std::string&& message) return writer.finish(); } -void -Journal::setRootPath(std::string_view fullPath, std::string_view relativePath) -{ - if (relativePath.size() <= fullPath.size() && - fullPath.ends_with(relativePath)) - { - m_filePathOffset_ = fullPath.size() - relativePath.size(); - } - else - { - m_filePathOffset_ = 0; - } -} void Journal::enableStructuredJournal() diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index fb2dcda5fdb..fe631b4100e 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -308,10 +308,12 @@ TEST_CASE("Test setJsonValue") writer.startObject(); log::detail::setJsonValue(writer, "testBool", true, &stringBuf); - log::detail::setJsonValue(writer, "testInt32", -1, &stringBuf); + log::detail::setJsonValue( + writer, "testInt32", -1, &stringBuf); log::detail::setJsonValue( writer, "testUInt32", 1, &stringBuf); - log::detail::setJsonValue(writer, "testInt64", -1, &stringBuf); + log::detail::setJsonValue( + writer, "testInt64", -1, &stringBuf); log::detail::setJsonValue( writer, "testUInt64", 1, &stringBuf); log::detail::setJsonValue(writer, "testDouble", 1.1, &stringBuf); @@ -327,7 +329,9 @@ TEST_CASE("Test setJsonValue") writer, "testStream", {}, &stringBuf); writer.endObject(); - CHECK(writer.finish() == R"AAA({"testBool":true,"testInt32":-1,"testUInt32":1,"testInt64":-1,"testUInt64":1,"testDouble":1.1,"testCharStar":"Char*","testStdString":"StdString","testStdStringView":"StdStringView","testToChars":"0","testStream":"0"})AAA"); + CHECK( + writer.finish() == + R"AAA({"testBool":true,"testInt32":-1,"testUInt32":1,"testInt64":-1,"testUInt64":1,"testDouble":1.1,"testCharStar":"Char*","testStdString":"StdString","testStdStringView":"StdStringView","testToChars":"0","testStream":"0"})AAA"); } TEST_CASE("Test json logging not enabled") @@ -421,30 +425,24 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log fields") CHECK(logValue.is_object()); CHECK(logValue.as_object().contains("GlobalParams")); CHECK(logValue.as_object().contains("JournalParams")); - CHECK(logValue.as_object().contains("MessageParams")); + CHECK(logValue.as_object().contains("Metadata")); CHECK(logValue.as_object().contains("Message")); CHECK(logValue.as_object()["GlobalParams"].is_object()); CHECK(logValue.as_object()["JournalParams"].is_object()); - CHECK(logValue.as_object()["MessageParams"].is_object()); + CHECK(logValue.as_object()["Metadata"].is_object()); CHECK(logValue.as_object()["Message"].is_string()); - CHECK( - logValue.as_object()["MessageParams"].as_object().contains("Function")); - CHECK(logValue.as_object()["MessageParams"].as_object().contains("File")); - CHECK(logValue.as_object()["MessageParams"].as_object().contains("Line")); - CHECK( - logValue.as_object()["MessageParams"].as_object().contains("ThreadId")); - CHECK(logValue.as_object()["MessageParams"].as_object().contains("Level")); - CHECK(logValue.as_object()["MessageParams"].as_object().contains("Time")); + CHECK(logValue.as_object()["Metadata"].as_object().contains("Function")); + CHECK(logValue.as_object()["Metadata"].as_object().contains("File")); + CHECK(logValue.as_object()["Metadata"].as_object().contains("Line")); + CHECK(logValue.as_object()["Metadata"].as_object().contains("ThreadId")); + CHECK(logValue.as_object()["Metadata"].as_object().contains("Level")); + CHECK(logValue.as_object()["Metadata"].as_object().contains("Time")); - CHECK(logValue.as_object()["MessageParams"] - .as_object()["Function"] - .is_string()); - CHECK( - logValue.as_object()["MessageParams"].as_object()["File"].is_string()); - CHECK( - logValue.as_object()["MessageParams"].as_object()["Line"].is_number()); + CHECK(logValue.as_object()["Metadata"].as_object()["Function"].is_string()); + CHECK(logValue.as_object()["Metadata"].as_object()["File"].is_string()); + CHECK(logValue.as_object()["Metadata"].as_object()["Line"].is_number()); CHECK( logValue.as_object()["Message"].get_string() == @@ -462,7 +460,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log levels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["MessageParams"] + logValue.as_object()["Metadata"] .as_object()["Level"] .get_string() == beast::severities::to_string(beast::severities::kTrace)); @@ -477,7 +475,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log levels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["MessageParams"] + logValue.as_object()["Metadata"] .as_object()["Level"] .get_string() == beast::severities::to_string(beast::severities::kDebug)); @@ -492,7 +490,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log levels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["MessageParams"] + logValue.as_object()["Metadata"] .as_object()["Level"] .get_string() == beast::severities::to_string(beast::severities::kInfo)); @@ -507,7 +505,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log levels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["MessageParams"] + logValue.as_object()["Metadata"] .as_object()["Level"] .get_string() == beast::severities::to_string(beast::severities::kWarning)); @@ -522,7 +520,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log levels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["MessageParams"] + logValue.as_object()["Metadata"] .as_object()["Level"] .get_string() == beast::severities::to_string(beast::severities::kError)); @@ -537,7 +535,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log levels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["MessageParams"] + logValue.as_object()["Metadata"] .as_object()["Level"] .get_string() == beast::severities::to_string(beast::severities::kFatal)); @@ -553,9 +551,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log stream") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["MessageParams"] - .as_object()["Level"] - .get_string() == + logValue.as_object()["Metadata"].as_object()["Level"].get_string() == beast::severities::to_string(beast::severities::kError)); } diff --git a/src/xrpld/app/main/Application.cpp b/src/xrpld/app/main/Application.cpp index 5351a906514..397261f4178 100644 --- a/src/xrpld/app/main/Application.cpp +++ b/src/xrpld/app/main/Application.cpp @@ -1213,10 +1213,6 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline) << ", Instance Cookie: " << log::param("InstanceCookie", instanceCookie_); - beast::Journal::addGlobalAttributes(log::attributes( - log::attr("RippledVersion", BuildInfo::getFullVersionString()), - log::attr("InstanceCookie", to_string(instanceCookie_)))); - if (numberOfThreads(*config_) < 2) { JLOG(m_journal.warn()) << "Limited to a single I/O service thread by " diff --git a/src/xrpld/app/main/Main.cpp b/src/xrpld/app/main/Main.cpp index 3bad24de440..115bb792674 100644 --- a/src/xrpld/app/main/Main.cpp +++ b/src/xrpld/app/main/Main.cpp @@ -796,11 +796,7 @@ run(int argc, char** argv) if (config->LOG_STYLE == LogStyle::Json) { - beast::Journal::setRootPath(std::source_location::current().file_name(), "src/xrpld/app/main/Main.cpp"); beast::Journal::enableStructuredJournal(); - beast::Journal::addGlobalAttributes(log::attributes( - log::attr("Application", "rippled"), - log::attr("NetworkID", config->NETWORK_ID))); } auto logs = std::make_unique(thresh); From 79c3a830889ef4c876afbfb2b1e8502e1d98d339 Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 4 Sep 2025 17:13:09 +0100 Subject: [PATCH 62/88] Fix build error Signed-off-by: JCW --- include/xrpl/basics/Log.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/xrpl/basics/Log.h b/include/xrpl/basics/Log.h index 2bf6e6a76d7..c2a66efb79b 100644 --- a/include/xrpl/basics/Log.h +++ b/include/xrpl/basics/Log.h @@ -154,7 +154,7 @@ class Logs // Batching members mutable std::mutex batchMutex_; static constexpr size_t BATCH_BUFFER_SIZE = 64 * 1024; // 64KB buffer - std::array batchBuffer_; + std::array batchBuffer_{}; std::span writeBuffer_; // Points to available write space std::span readBuffer_; // Points to data ready to flush std::chrono::steady_clock::time_point lastFlush_ = From ce5a6aec7b6d3a94c45a786a26762f4929f7bba3 Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 4 Sep 2025 20:42:50 +0100 Subject: [PATCH 63/88] Revert unneeded changes Signed-off-by: JCW --- src/xrpld/app/consensus/RCLConsensus.cpp | 30 +++++++++++++++++------- src/xrpld/app/main/Application.cpp | 1 + src/xrpld/app/misc/NetworkOPs.cpp | 8 ------- src/xrpld/app/tx/detail/Transactor.cpp | 7 +----- src/xrpld/app/tx/detail/Transactor.h | 13 +++------- src/xrpld/overlay/detail/PeerImp.cpp | 5 ---- src/xrpld/overlay/detail/PeerImp.h | 12 ++++------ 7 files changed, 30 insertions(+), 46 deletions(-) diff --git a/src/xrpld/app/consensus/RCLConsensus.cpp b/src/xrpld/app/consensus/RCLConsensus.cpp index 431b9e6bc24..3a872bf9a5c 100644 --- a/src/xrpld/app/consensus/RCLConsensus.cpp +++ b/src/xrpld/app/consensus/RCLConsensus.cpp @@ -1109,11 +1109,7 @@ RclConsensusLogger::RclConsensusLogger( bool const validating, beast::Journal j, std::source_location location) - : j_(j, - log::attributes( - log::attr("Role", "ConsensusLogger"), - log::attr("Label", label))) - , location_(location) + : j_(j) { if (!validating && !j.info()) return; @@ -1131,10 +1127,26 @@ RclConsensusLogger::~RclConsensusLogger() auto const duration = std::chrono::duration_cast( std::chrono::steady_clock::now() - start_); - j_.info(location_) << header_ << "duration " << (duration.count() / 1000) - << '.' << std::setw(3) << std::setfill('0') - << (duration.count() % 1000) << "s. " << ss_->str() - << log::field("Duration", duration.count()); + std::stringstream outSs; + outSs << header_ << "duration " << (duration.count() / 1000) << '.' + << std::setw(3) << std::setfill('0') << (duration.count() % 1000) + << "s. " << ss_->str(); + + if (beast::Journal::isStructuredJournalEnabled()) + { + thread_local std::string buffer; + buffer.reserve(1024); + beast::detail::SimpleJsonWriter writer{buffer}; + writer.startObject(); + writer.writeKey("Message"); + writer.writeString(outSs.str()); + writer.endObject(); + j_.sink().writeAlways(beast::severities::kInfo, std::string{writer.finish()}); + } + else + { + j_.sink().writeAlways(beast::severities::kInfo, outSs.str()); + } } } // namespace ripple diff --git a/src/xrpld/app/main/Application.cpp b/src/xrpld/app/main/Application.cpp index 397261f4178..c7cb6d07972 100644 --- a/src/xrpld/app/main/Application.cpp +++ b/src/xrpld/app/main/Application.cpp @@ -1644,6 +1644,7 @@ ApplicationImp::run() perfLog_->stop(); JLOG(m_journal.info()) << "Done."; + logs_->flushBatch(); } void diff --git a/src/xrpld/app/misc/NetworkOPs.cpp b/src/xrpld/app/misc/NetworkOPs.cpp index b99ae7d6a68..b9069442f8c 100644 --- a/src/xrpld/app/misc/NetworkOPs.cpp +++ b/src/xrpld/app/misc/NetworkOPs.cpp @@ -1190,10 +1190,6 @@ NetworkOPsImp::strOperatingMode(OperatingMode const mode, bool const admin) void NetworkOPsImp::submitTransaction(std::shared_ptr const& iTrans) { - beast::Journal journal{ - m_journal, - log::attributes( - log::attr("TransactionID", to_string(iTrans->getTransactionID())))}; if (isNeedNetworkLedger()) { // Nothing we can do if we've never been in sync @@ -1257,10 +1253,6 @@ NetworkOPsImp::submitTransaction(std::shared_ptr const& iTrans) bool NetworkOPsImp::preProcessTransaction(std::shared_ptr& transaction) { - beast::Journal journal{ - m_journal, - log::attributes( - log::attr("TransactionID", to_string(transaction->getID())))}; auto const newFlags = app_.getHashRouter().getFlags(transaction->getID()); if ((newFlags & HashRouterFlags::BAD) != HashRouterFlags::UNDEFINED) diff --git a/src/xrpld/app/tx/detail/Transactor.cpp b/src/xrpld/app/tx/detail/Transactor.cpp index a48643b2baa..8f881d7252f 100644 --- a/src/xrpld/app/tx/detail/Transactor.cpp +++ b/src/xrpld/app/tx/detail/Transactor.cpp @@ -206,12 +206,7 @@ preflight2(PreflightContext const& ctx) //------------------------------------------------------------------------------ Transactor::Transactor(ApplyContext& ctx) - : ctx_(ctx) - , account_(ctx.tx.getAccountID(sfAccount)) - , j_(ctx.journal, - log::attributes( - log::attr("TransactionID", to_string(ctx_.tx.getTransactionID())), - log::attr("AccountID", to_string(account_)))) + : ctx_(ctx), j_(ctx.journal), account_(ctx.tx.getAccountID(sfAccount)) { } diff --git a/src/xrpld/app/tx/detail/Transactor.h b/src/xrpld/app/tx/detail/Transactor.h index f95d396b908..fc58a50b844 100644 --- a/src/xrpld/app/tx/detail/Transactor.h +++ b/src/xrpld/app/tx/detail/Transactor.h @@ -53,10 +53,7 @@ struct PreflightContext , rules(rules_) , flags(flags_) , parentBatchId(parentBatchId_) - , j(j_, - log::attributes( - log::attr("TransactionID", to_string(tx.getTransactionID())), - log::attr("AccountID", to_string(tx.getAccountID(sfAccount))))) + , j(j_) { XRPL_ASSERT( (flags_ & tapBATCH) == tapBATCH, "Batch apply flag should be set"); @@ -104,10 +101,7 @@ struct PreclaimContext , flags(flags_) , tx(tx_) , parentBatchId(parentBatchId_) - , j(j_, - log::attributes( - log::attr("TransactionID", to_string(tx.getTransactionID())), - log::attr("AccountID", to_string(tx.getAccountID(sfAccount))))) + , j(j_) { XRPL_ASSERT( parentBatchId.has_value() == ((flags_ & tapBATCH) == tapBATCH), @@ -145,13 +139,12 @@ class Transactor { protected: ApplyContext& ctx_; + beast::Journal const j_; AccountID const account_; XRPAmount mPriorBalance; // Balance before fees. XRPAmount mSourceBalance; // Balance after fees. - beast::Journal const j_; - virtual ~Transactor() = default; Transactor(Transactor const&) = delete; Transactor& diff --git a/src/xrpld/overlay/detail/PeerImp.cpp b/src/xrpld/overlay/detail/PeerImp.cpp index 587e29b6052..bcb3a3cdaf2 100644 --- a/src/xrpld/overlay/detail/PeerImp.cpp +++ b/src/xrpld/overlay/detail/PeerImp.cpp @@ -1396,11 +1396,6 @@ PeerImp::handleTransaction( { auto stx = std::make_shared(sit); uint256 txID = stx->getTransactionID(); - beast::Journal protocolJournal{ - p_journal_, - log::attributes( - log::attr("TransactionID", to_string(txID)), - log::attr("RawTransaction", strHex(m->rawtransaction())))}; // Charge strongly for attempting to relay a txn with tfInnerBatchTxn // LCOV_EXCL_START diff --git a/src/xrpld/overlay/detail/PeerImp.h b/src/xrpld/overlay/detail/PeerImp.h index 7e8f96368b5..a7268fd26e1 100644 --- a/src/xrpld/overlay/detail/PeerImp.h +++ b/src/xrpld/overlay/detail/PeerImp.h @@ -831,18 +831,14 @@ PeerImp::PeerImp( app_.journal("Peer"), log::attributes( log::attr("NodeID", id), - log::attr("RemoteAddress", to_string(slot->remote_endpoint())), - log::attr( - "PublicKey", - toBase58(TokenType::NodePublic, publicKey)))) + log::attr("RemoteAddress", to_string(slot->remote_endpoint())) + )) , p_journal_( app_.journal("Protocol"), log::attributes( log::attr("NodeID", id), - log::attr("RemoteAddress", to_string(slot->remote_endpoint())), - log::attr( - "PublicKey", - toBase58(TokenType::NodePublic, publicKey)))) + log::attr("RemoteAddress", to_string(slot->remote_endpoint()) + ))) , stream_ptr_(std::move(stream_ptr)) , socket_(stream_ptr_->next_layer().socket()) , stream_(*stream_ptr_) From dcec5a0bbcc96e8df1f9591ffe2f41cca91dd23f Mon Sep 17 00:00:00 2001 From: JCW Date: Mon, 8 Sep 2025 16:40:34 +0100 Subject: [PATCH 64/88] Revert unrelated changes & performance optimisation Signed-off-by: JCW --- include/xrpl/basics/Log.h | 8 +- include/xrpl/beast/utility/Journal.h | 6 +- include/xrpl/beast/utility/WrappedSink.h | 8 +- include/xrpl/resource/detail/Logic.h | 29 +++---- src/libxrpl/basics/Log.cpp | 89 +++++++++++---------- src/libxrpl/beast/utility/beast_Journal.cpp | 10 +-- src/test/beast/beast_Journal_test.cpp | 4 +- src/test/csf/Sim.h | 4 +- src/test/jtx/CaptureLogs.h | 4 +- src/test/jtx/CheckMessageLogs.h | 4 +- src/test/server/Server_test.cpp | 4 +- src/test/unit_test/SuiteJournal.h | 12 +-- src/tests/libxrpl/basics/log.cpp | 10 +-- 13 files changed, 92 insertions(+), 100 deletions(-) diff --git a/include/xrpl/basics/Log.h b/include/xrpl/basics/Log.h index c2a66efb79b..dae3fa14afd 100644 --- a/include/xrpl/basics/Log.h +++ b/include/xrpl/basics/Log.h @@ -71,10 +71,10 @@ class Logs operator=(Sink const&) = delete; void - write(beast::severities::Severity level, std::string&& text) override; + write(beast::severities::Severity level, std::string_view text) override; void - writeAlways(beast::severities::Severity level, std::string&& text) + writeAlways(beast::severities::Severity level, std::string_view text) override; }; @@ -205,7 +205,7 @@ class Logs write( beast::severities::Severity level, std::string const& partition, - std::string const& text, + std::string_view text, bool console); std::string @@ -246,7 +246,7 @@ class Logs static void format( std::string& output, - std::string const& message, + std::string_view message, beast::severities::Severity severity, std::string const& partition); diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 0e02336033a..da73a0a3e07 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -366,7 +366,7 @@ class Journal severities::Severity severity) const; static std::string_view - formatLog(std::string&& message); + formatLog(std::string const& message); public: //-------------------------------------------------------------------------- @@ -420,7 +420,7 @@ class Journal level is below the current threshold(). */ virtual void - write(Severity level, std::string&& text) = 0; + write(Severity level, std::string_view text) = 0; /** Bypass filter and write text to the sink at the specified severity. * Always write the message, but maintain the same formatting as if @@ -430,7 +430,7 @@ class Journal * @param text Text to write to sink. */ virtual void - writeAlways(Severity level, std::string&& text) = 0; + writeAlways(Severity level, std::string_view text) = 0; private: Severity thresh_; diff --git a/include/xrpl/beast/utility/WrappedSink.h b/include/xrpl/beast/utility/WrappedSink.h index ee8641849d3..e1343dc9e69 100644 --- a/include/xrpl/beast/utility/WrappedSink.h +++ b/include/xrpl/beast/utility/WrappedSink.h @@ -88,17 +88,17 @@ class WrappedSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, std::string&& text) override + write(beast::severities::Severity level, std::string_view text) override { using beast::Journal; - sink_.write(level, prefix_ + text); + sink_.write(level, prefix_ + std::string{text}); } void - writeAlways(severities::Severity level, std::string&& text) override + writeAlways(severities::Severity level, std::string_view text) override { using beast::Journal; - sink_.writeAlways(level, prefix_ + text); + sink_.writeAlways(level, prefix_ + std::string{text}); } }; diff --git a/include/xrpl/resource/detail/Logic.h b/include/xrpl/resource/detail/Logic.h index 8bfa2c2ac83..b07ee00e73a 100644 --- a/include/xrpl/resource/detail/Logic.h +++ b/include/xrpl/resource/detail/Logic.h @@ -132,8 +132,7 @@ class Logic } } - JLOG(m_journal.debug()) - << "New inbound endpoint " << log::param("Entry", *entry); + JLOG(m_journal.debug()) << "New inbound endpoint " << *entry; return Consumer(*this, *entry); } @@ -161,8 +160,7 @@ class Logic } } - JLOG(m_journal.debug()) - << "New outbound endpoint " << log::param("Entry", *entry); + JLOG(m_journal.debug()) << "New outbound endpoint " << *entry; return Consumer(*this, *entry); } @@ -195,8 +193,7 @@ class Logic } } - JLOG(m_journal.debug()) - << "New unlimited endpoint " << log::param("Entry", *entry); + JLOG(m_journal.debug()) << "New unlimited endpoint " << *entry; return Consumer(*this, *entry); } @@ -353,8 +350,7 @@ class Logic { if (iter->whenExpires <= elapsed) { - JLOG(m_journal.debug()) - << "Expired " << log::param("Entry", *iter); + JLOG(m_journal.debug()) << "Expired " << *iter; auto table_iter = table_.find(*iter->key); ++iter; erase(table_iter); @@ -426,9 +422,7 @@ class Logic std::lock_guard _(lock_); if (--entry.refcount == 0) { - JLOG(m_journal.debug()) - << "Inactive " << log::param("Entry", entry); - ; + JLOG(m_journal.debug()) << "Inactive " << entry; switch (entry.key->kind) { @@ -480,8 +474,7 @@ class Logic clock_type::time_point const now(m_clock.now()); int const balance(entry.add(fee.cost(), now)); JLOG(getStream(fee.cost(), m_journal)) - << "Charging " << log::param("Entry", entry) << " for " - << log::param("Fee", fee) << context; + << "Charging " << entry << " for " << fee << context; return disposition(balance); } @@ -503,9 +496,7 @@ class Logic } if (notify) { - JLOG(m_journal.info()) - << "Load warning: " << log::param("Entry", entry); - ; + JLOG(m_journal.info()) << "Load warning: " << entry; ++m_stats.warn; } return notify; @@ -524,10 +515,8 @@ class Logic if (balance >= dropThreshold) { JLOG(m_journal.warn()) - << "Consumer entry " << log::param("Entry", entry) - << " dropped with balance " << log::param("Entry", balance) - << " at or above drop threshold " - << log::param("Entry", dropThreshold); + << "Consumer entry " << entry << " dropped with balance " + << balance << " at or above drop threshold " << dropThreshold; // Adding feeDrop at this point keeps the dropped connection // from re-connecting for at least a little while after it is diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index 996c8bf3f11..cdb4b5820b8 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -52,18 +52,18 @@ Logs::Sink::Sink( } void -Logs::Sink::write(beast::severities::Severity level, std::string&& text) +Logs::Sink::write(beast::severities::Severity level, std::string_view text) { if (level < threshold()) return; - logs_.write(level, partition_, std::move(text), console()); + logs_.write(level, partition_, text, console()); } void -Logs::Sink::writeAlways(beast::severities::Severity level, std::string&& text) +Logs::Sink::writeAlways(beast::severities::Severity level, std::string_view text) { - logs_.write(level, partition_, std::move(text), console()); + logs_.write(level, partition_, text, console()); } //------------------------------------------------------------------------------ @@ -190,21 +190,26 @@ void Logs::write( beast::severities::Severity level, std::string const& partition, - std::string const& text, + std::string_view text, bool console) { std::string s; - format(s, text, level, partition); + std::string_view result = text; + if (!beast::Journal::isStructuredJournalEnabled()) + { + format(s, text, level, partition); + result = s; + } // Console output still immediate for responsiveness if (!silent_) - std::cerr << s << '\n'; + std::cerr << result << '\n'; // Add to batch buffer for file output { std::lock_guard lock(batchMutex_); - size_t logSize = s.size() + 1; // +1 for newline + size_t logSize = result.size() + 1; // +1 for newline // If log won't fit in current write buffer, flush first if (logSize > writeBuffer_.size()) @@ -213,8 +218,8 @@ Logs::write( } // Copy log into write buffer - std::copy(s.begin(), s.end(), writeBuffer_.begin()); - writeBuffer_[s.size()] = '\n'; + std::copy(result.begin(), result.end(), writeBuffer_.begin()); + writeBuffer_[result.size()] = '\n'; // Update spans: expand read buffer, shrink write buffer size_t totalUsed = readBuffer_.size() + logSize; @@ -378,47 +383,45 @@ Logs::fromString(std::string const& s) void Logs::format( std::string& output, - std::string const& message, + std::string_view message, beast::severities::Severity severity, std::string const& partition) { output = message; - if (!beast::Journal::isStructuredJournalEnabled()) - { - output.reserve(output.size() + partition.size() + 100); - output += to_string(std::chrono::system_clock::now()); + output.reserve(output.size() + partition.size() + 100); + output += to_string(std::chrono::system_clock::now()); - output += " "; - if (!partition.empty()) - output += partition + ":"; + output += " "; + if (!partition.empty()) + output += partition + ":"; - using namespace beast::severities; - switch (severity) - { - case kTrace: - output += "TRC "; - break; - case kDebug: - output += "DBG "; - break; - case kInfo: - output += "NFO "; - break; - case kWarning: - output += "WRN "; - break; - case kError: - output += "ERR "; - break; - default: - UNREACHABLE("ripple::Logs::format : invalid severity"); - [[fallthrough]]; - case kFatal: - output += "FTL "; - break; - } + using namespace beast::severities; + switch (severity) + { + case kTrace: + output += "TRC "; + break; + case kDebug: + output += "DBG "; + break; + case kInfo: + output += "NFO "; + break; + case kWarning: + output += "WRN "; + break; + case kError: + output += "ERR "; + break; + default: + UNREACHABLE("ripple::Logs::format : invalid severity"); + [[fallthrough]]; + case kFatal: + output += "FTL "; + break; } + // Limit the maximum length of the output if (output.size() > maximumMessageCharacters) { diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index 88ada9664bf..8c4663ad290 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -156,12 +156,12 @@ class NullJournalSink : public Journal::Sink } void - write(severities::Severity, std::string&&) override + write(severities::Severity, std::string_view) override { } void - writeAlways(severities::Severity, std::string&&) override + writeAlways(severities::Severity, std::string_view) override { } }; @@ -293,7 +293,7 @@ Journal::initMessageContext( } std::string_view -Journal::formatLog(std::string&& message) +Journal::formatLog(std::string const& message) { if (!m_jsonLogsEnabled) { @@ -391,9 +391,9 @@ Journal::ScopedStream::~ScopedStream() if (!s.empty()) { if (s == "\n") - m_sink.write(m_level, std::string{formatLog("")}); + m_sink.write(m_level, formatLog("")); else - m_sink.write(m_level, std::string{formatLog(std::move(s))}); + m_sink.write(m_level, formatLog(s)); } } diff --git a/src/test/beast/beast_Journal_test.cpp b/src/test/beast/beast_Journal_test.cpp index abd4af4f8dd..b1379a34563 100644 --- a/src/test/beast/beast_Journal_test.cpp +++ b/src/test/beast/beast_Journal_test.cpp @@ -48,14 +48,14 @@ class Journal_test : public unit_test::suite } void - write(severities::Severity level, std::string&&) override + write(severities::Severity level, std::string_view) override { if (level >= threshold()) ++m_count; } void - writeAlways(severities::Severity level, std::string&&) override + writeAlways(severities::Severity level, std::string_view) override { ++m_count; } diff --git a/src/test/csf/Sim.h b/src/test/csf/Sim.h index b61cbbeb1ac..313f8abd8ff 100644 --- a/src/test/csf/Sim.h +++ b/src/test/csf/Sim.h @@ -49,7 +49,7 @@ class BasicSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, std::string&& text) override + write(beast::severities::Severity level, std::string_view text) override { if (level < threshold()) return; @@ -59,7 +59,7 @@ class BasicSink : public beast::Journal::Sink } void - writeAlways(beast::severities::Severity level, std::string&& text) override + writeAlways(beast::severities::Severity level, std::string_view text) override { std::cout << clock_.now().time_since_epoch().count() << " " << text << std::endl; diff --git a/src/test/jtx/CaptureLogs.h b/src/test/jtx/CaptureLogs.h index c89f1f74fac..4fffae130ef 100644 --- a/src/test/jtx/CaptureLogs.h +++ b/src/test/jtx/CaptureLogs.h @@ -57,14 +57,14 @@ class CaptureLogs : public Logs } void - write(beast::severities::Severity level, std::string&& text) override + write(beast::severities::Severity level, std::string_view text) override { std::lock_guard lock(strmMutex_); strm_ << text; } void - writeAlways(beast::severities::Severity level, std::string&& text) + writeAlways(beast::severities::Severity level, std::string_view text) override { std::lock_guard lock(strmMutex_); diff --git a/src/test/jtx/CheckMessageLogs.h b/src/test/jtx/CheckMessageLogs.h index e05ac5e4cfb..6b950a82eeb 100644 --- a/src/test/jtx/CheckMessageLogs.h +++ b/src/test/jtx/CheckMessageLogs.h @@ -45,14 +45,14 @@ class CheckMessageLogs : public Logs } void - write(beast::severities::Severity level, std::string&& text) override + write(beast::severities::Severity level, std::string_view text) override { if (text.find(owner_.msg_) != std::string::npos) *owner_.pFound_ = true; } void - writeAlways(beast::severities::Severity level, std::string&& text) + writeAlways(beast::severities::Severity level, std::string_view text) override { write(level, std::move(text)); diff --git a/src/test/server/Server_test.cpp b/src/test/server/Server_test.cpp index db1c58c2bc1..1c5aa67992c 100644 --- a/src/test/server/Server_test.cpp +++ b/src/test/server/Server_test.cpp @@ -92,7 +92,7 @@ class Server_test : public beast::unit_test::suite } void - write(beast::severities::Severity level, std::string&& text) override + write(beast::severities::Severity level, std::string_view text) override { if (level < threshold()) return; @@ -101,7 +101,7 @@ class Server_test : public beast::unit_test::suite } void - writeAlways(beast::severities::Severity level, std::string&& text) + writeAlways(beast::severities::Severity level, std::string_view text) override { suite_.log << text << std::endl; diff --git a/src/test/unit_test/SuiteJournal.h b/src/test/unit_test/SuiteJournal.h index 8fb0f122bdc..e58e435e6f1 100644 --- a/src/test/unit_test/SuiteJournal.h +++ b/src/test/unit_test/SuiteJournal.h @@ -49,14 +49,14 @@ class SuiteJournalSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, std::string&& text) override; + write(beast::severities::Severity level, std::string_view text) override; void - writeAlways(beast::severities::Severity level, std::string&& text) override; + writeAlways(beast::severities::Severity level, std::string_view text) override; }; inline void -SuiteJournalSink::write(beast::severities::Severity level, std::string&& text) +SuiteJournalSink::write(beast::severities::Severity level, std::string_view text) { // Only write the string if the level at least equals the threshold. if (level >= threshold()) @@ -66,7 +66,7 @@ SuiteJournalSink::write(beast::severities::Severity level, std::string&& text) inline void SuiteJournalSink::writeAlways( beast::severities::Severity level, - std::string&& text) + std::string_view text) { using namespace beast::severities; @@ -134,7 +134,7 @@ class StreamSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, std::string&& text) override + write(beast::severities::Severity level, std::string_view text) override { if (level < threshold()) return; @@ -142,7 +142,7 @@ class StreamSink : public beast::Journal::Sink } inline void - writeAlways(beast::severities::Severity level, std::string&& text) override + writeAlways(beast::severities::Severity level, std::string_view text) override { strm_ << text << std::endl; } diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index fe631b4100e..4ad24a896d7 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -53,13 +53,13 @@ class MockLogs : public Logs operator=(Sink const&) = delete; void - write(beast::severities::Severity level, std::string&& text) override + write(beast::severities::Severity level, std::string_view text) override { logs_.write(level, partition_, text, false); } void - writeAlways(beast::severities::Severity level, std::string&& text) + writeAlways(beast::severities::Severity level, std::string_view text) override { logs_.write(level, partition_, text, false); @@ -86,7 +86,7 @@ class MockLogs : public Logs write( beast::severities::Severity level, std::string const& partition, - std::string const& text, + std::string_view text, bool console) { std::string s; @@ -365,13 +365,13 @@ class MockSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, std::string&& text) override + write(beast::severities::Severity level, std::string_view text) override { strm_ << text; } void - writeAlways(beast::severities::Severity level, std::string&& text) override + writeAlways(beast::severities::Severity level, std::string_view text) override { strm_ << text; } From 2c2936fa93de5543c501c53237efc62bc7528d1d Mon Sep 17 00:00:00 2001 From: JCW Date: Mon, 8 Sep 2025 17:46:23 +0100 Subject: [PATCH 65/88] Fix issues Signed-off-by: JCW --- src/libxrpl/basics/Log.cpp | 7 ++++--- src/tests/libxrpl/basics/log.cpp | 21 +++++++-------------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index cdb4b5820b8..b8463b15104 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -387,9 +387,9 @@ Logs::format( beast::severities::Severity severity, std::string const& partition) { - output = message; - output.reserve(output.size() + partition.size() + 100); - output += to_string(std::chrono::system_clock::now()); + output.reserve(message.size() + partition.size() + 100); + + output = to_string(std::chrono::system_clock::now()); output += " "; if (!partition.empty()) @@ -421,6 +421,7 @@ Logs::format( break; } + output += message; // Limit the maximum length of the output if (output.size() > maximumMessageCharacters) diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index 4ad24a896d7..f0886d4c6e0 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -90,8 +90,13 @@ class MockLogs : public Logs bool console) { std::string s; - format(s, text, level, partition); - logStream_.append(s); + std::string_view result = text; + if (!beast::Journal::isStructuredJournalEnabled()) + { + format(s, text, level, partition); + result = s; + } + logStream_.append(result); } }; @@ -120,18 +125,6 @@ TEST_CASE("Test format output") CHECK(output != "Message"); } -TEST_CASE("Test format output when structured logs are enabled") -{ - beast::Journal::enableStructuredJournal(); - - std::string output; - Logs::format(output, "Message", beast::severities::kDebug, "Test"); - - CHECK(output == "Message"); - - beast::Journal::disableStructuredJournal(); -} - TEST_CASE("Enable json logs") { std::string logStream; From 1dc3b256e06d62db250c4bd430f57288a8e5f4a6 Mon Sep 17 00:00:00 2001 From: JCW Date: Tue, 9 Sep 2025 16:33:17 +0100 Subject: [PATCH 66/88] Bugfix Signed-off-by: JCW --- src/xrpld/app/consensus/RCLConsensus.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/xrpld/app/consensus/RCLConsensus.cpp b/src/xrpld/app/consensus/RCLConsensus.cpp index 3a872bf9a5c..ed54f770b09 100644 --- a/src/xrpld/app/consensus/RCLConsensus.cpp +++ b/src/xrpld/app/consensus/RCLConsensus.cpp @@ -1136,12 +1136,13 @@ RclConsensusLogger::~RclConsensusLogger() { thread_local std::string buffer; buffer.reserve(1024); + buffer.clear(); beast::detail::SimpleJsonWriter writer{buffer}; writer.startObject(); writer.writeKey("Message"); writer.writeString(outSs.str()); writer.endObject(); - j_.sink().writeAlways(beast::severities::kInfo, std::string{writer.finish()}); + j_.sink().writeAlways(beast::severities::kInfo, writer.finish()); } else { From d33691da84fad5bd9eecc438ac37f6222a933226 Mon Sep 17 00:00:00 2001 From: JCW Date: Wed, 10 Sep 2025 01:14:16 +0100 Subject: [PATCH 67/88] Log size optimise Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 23 +++++++++++++ src/libxrpl/basics/Log.cpp | 8 ++--- src/libxrpl/beast/utility/beast_Journal.cpp | 38 +++++++++------------ src/xrpld/app/consensus/RCLConsensus.cpp | 4 ++- 4 files changed, 46 insertions(+), 27 deletions(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index da73a0a3e07..7a15dc6d07e 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -324,6 +324,7 @@ class Journal { std::string buffer_; detail::SimpleJsonWriter messageParamsWriter_; + bool hasMessageParams_ = false; public: JsonLogContext() : messageParamsWriter_(buffer_) @@ -331,6 +332,26 @@ class Journal buffer_.reserve(1024 * 5); } + void + startMessageParams() + { + if (!hasMessageParams_) + { + writer().writeKey("Data"); + writer().startObject(); + hasMessageParams_ = true; + } + } + + void + endMessageParams() + { + if (hasMessageParams_) + { + writer().endObject(); + } + } + detail::SimpleJsonWriter& writer() { @@ -1014,6 +1035,7 @@ operator<<(std::ostream& os, LogParameter const& param) os << param.value_; return os; } + beast::Journal::currentJsonLogContext_.startMessageParams(); detail::setJsonValue( beast::Journal::currentJsonLogContext_.writer(), param.name_, @@ -1028,6 +1050,7 @@ operator<<(std::ostream& os, LogField const& param) { if (!beast::Journal::m_jsonLogsEnabled) return os; + beast::Journal::currentJsonLogContext_.startMessageParams(); detail::setJsonValue( beast::Journal::currentJsonLogContext_.writer(), param.name_, diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index b8463b15104..943951dec5b 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -201,14 +201,14 @@ Logs::write( result = s; } - // Console output still immediate for responsiveness - if (!silent_) - std::cerr << result << '\n'; - // Add to batch buffer for file output { std::lock_guard lock(batchMutex_); + // Console output still immediate for responsiveness + if (!silent_) + std::cerr << result << '\n'; + size_t logSize = result.size() + 1; // +1 for newline // If log won't fit in current write buffer, flush first diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index 8c4663ad290..20236e1a5a0 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -216,7 +216,6 @@ Journal::JsonLogContext::reset( ThreadIdStringInitializer() { std::stringstream threadIdStream; - threadIdStream.imbue(std::locale::classic()); threadIdStream << std::this_thread::get_id(); value = threadIdStream.str(); } @@ -229,7 +228,7 @@ Journal::JsonLogContext::reset( if (!journalAttributesJson.empty()) { - writer().writeKey("JournalParams"); + writer().writeKey("Jnl"); writer().writeRaw(journalAttributesJson); writer().endObject(); } @@ -238,50 +237,45 @@ Journal::JsonLogContext::reset( std::shared_lock lock(globalLogAttributesMutex_); if (!globalLogAttributesJson_.empty()) { - writer().writeKey("GlobalParams"); + writer().writeKey("Gbl"); writer().writeRaw(globalLogAttributesJson_); writer().endObject(); } } - writer().writeKey("ModuleName"); - writer().writeString(moduleName); - writer().writeKey("Metadata"); + writer().writeKey("Mtd"); writer().startObject(); - writer().writeKey("Function"); - writer().writeString(location.function_name()); + writer().writeKey("Mdl"); + writer().writeString(moduleName); - writer().writeKey("File"); + writer().writeKey("Fl"); + constexpr size_t FILE_NAME_KEEP_CHARS = 20; std::string_view fileName = location.file_name(); - constexpr size_t KEEP_CHARS = 10; - std::string_view trimmedFileName = (fileName.size() > KEEP_CHARS) - ? fileName.substr(fileName.size() - KEEP_CHARS) + std::string_view trimmedFileName = (fileName.size() > FILE_NAME_KEEP_CHARS) + ? fileName.substr(fileName.size() - FILE_NAME_KEEP_CHARS) : fileName; writer().writeString(trimmedFileName); - writer().writeKey("Line"); + writer().writeKey("Ln"); writer().writeUInt(location.line()); - writer().writeKey("ThreadId"); + writer().writeKey("ThId"); writer().writeString(threadId.value); auto severityStr = to_string(severity); - writer().writeKey("Level"); + writer().writeKey("Lv"); writer().writeString(severityStr); auto nowMs = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); - writer().writeKey("Timestamp"); - writer().writeInt(nowMs); - writer().writeKey("Time"); + writer().writeKey("Tm"); writer().writeString(fastTimestampToString(nowMs)); writer().endObject(); - writer().writeKey("MessageParams"); - writer().startObject(); + hasMessageParams_ = false; } void @@ -302,9 +296,9 @@ Journal::formatLog(std::string const& message) auto& writer = currentJsonLogContext_.writer(); - writer.endObject(); + currentJsonLogContext_.endMessageParams(); - writer.writeKey("Message"); + writer.writeKey("Msg"); writer.writeString(message); writer.endObject(); diff --git a/src/xrpld/app/consensus/RCLConsensus.cpp b/src/xrpld/app/consensus/RCLConsensus.cpp index ed54f770b09..4e13edc649a 100644 --- a/src/xrpld/app/consensus/RCLConsensus.cpp +++ b/src/xrpld/app/consensus/RCLConsensus.cpp @@ -1139,8 +1139,10 @@ RclConsensusLogger::~RclConsensusLogger() buffer.clear(); beast::detail::SimpleJsonWriter writer{buffer}; writer.startObject(); - writer.writeKey("Message"); + writer.writeKey("Msg"); writer.writeString(outSs.str()); + writer.writeKey("Tm"); + writer.writeString(to_string(std::chrono::system_clock::now())); writer.endObject(); j_.sink().writeAlways(beast::severities::kInfo, writer.finish()); } From f44d53be16254d3f2323528d2378bd81e1625579 Mon Sep 17 00:00:00 2001 From: JCW Date: Wed, 10 Sep 2025 07:42:17 +0100 Subject: [PATCH 68/88] Fix test error Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 2 +- src/libxrpl/beast/utility/beast_Journal.cpp | 2 +- src/tests/libxrpl/basics/log.cpp | 192 ++++++++++---------- 3 files changed, 97 insertions(+), 99 deletions(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 7a15dc6d07e..9a0b5436bb6 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -337,7 +337,7 @@ class Journal { if (!hasMessageParams_) { - writer().writeKey("Data"); + writer().writeKey("Dt"); writer().startObject(); hasMessageParams_ = true; } diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index 20236e1a5a0..89de4baaf9d 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -237,7 +237,7 @@ Journal::JsonLogContext::reset( std::shared_lock lock(globalLogAttributesMutex_); if (!globalLogAttributesJson_.empty()) { - writer().writeKey("Gbl"); + writer().writeKey("Glb"); writer().writeRaw(globalLogAttributesJson_); writer().endObject(); } diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index f0886d4c6e0..d550c7f0401 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -120,9 +120,9 @@ TEST_CASE("Text logs") TEST_CASE("Test format output") { std::string output; - Logs::format(output, "Message", beast::severities::kDebug, "Test"); - CHECK(output.find("Message") != std::string::npos); - CHECK(output != "Message"); + Logs::format(output, "Msg", beast::severities::kDebug, "Test"); + CHECK(output.find("Msg") != std::string::npos); + CHECK(output != "Msg"); } TEST_CASE("Enable json logs") @@ -146,9 +146,9 @@ TEST_CASE("Enable json logs") CHECK(ec == boost::system::errc::success); CHECK(doc.is_object()); - CHECK(doc.as_object().contains("Message")); - CHECK(doc.as_object()["Message"].is_string()); - CHECK(doc.as_object()["Message"].get_string() == ""); + CHECK(doc.as_object().contains("Msg")); + CHECK(doc.as_object()["Msg"].is_string()); + CHECK(doc.as_object()["Msg"].get_string() == ""); beast::Journal::disableStructuredJournal(); } @@ -169,13 +169,13 @@ TEST_CASE("Global attributes") CHECK(ec == boost::system::errc::success); CHECK(jsonLog.is_object()); - CHECK(jsonLog.as_object().contains("GlobalParams")); - CHECK(jsonLog.as_object()["GlobalParams"].is_object()); - CHECK(jsonLog.as_object()["GlobalParams"].as_object().contains("Field1")); + CHECK(jsonLog.as_object().contains("Glb")); + CHECK(jsonLog.as_object()["Glb"].is_object()); + CHECK(jsonLog.as_object()["Glb"].as_object().contains("Field1")); CHECK( - jsonLog.as_object()["GlobalParams"].as_object()["Field1"].is_string()); + jsonLog.as_object()["Glb"].as_object()["Field1"].is_string()); CHECK( - jsonLog.as_object()["GlobalParams"] + jsonLog.as_object()["Glb"] .as_object()["Field1"] .get_string() == "Value1"); beast::Journal::disableStructuredJournal(); @@ -203,19 +203,19 @@ TEST_CASE("Global attributes inheritable") CHECK(ec == boost::system::errc::success); CHECK(jsonLog.is_object()); - CHECK(jsonLog.as_object()["GlobalParams"].as_object().contains("Field1")); + CHECK(jsonLog.as_object()["Glb"].as_object().contains("Field1")); CHECK( - jsonLog.as_object()["GlobalParams"].as_object()["Field1"].is_string()); + jsonLog.as_object()["Glb"].as_object()["Field1"].is_string()); CHECK( - jsonLog.as_object()["GlobalParams"] + jsonLog.as_object()["Glb"] .as_object()["Field1"] .get_string() == "Value1"); CHECK( - jsonLog.as_object()["JournalParams"] + jsonLog.as_object()["Jnl"] .as_object()["Field1"] .get_string() == "Value3"); CHECK( - jsonLog.as_object()["JournalParams"] + jsonLog.as_object()["Jnl"] .as_object()["Field2"] .get_string() == "Value2"); beast::Journal::disableStructuredJournal(); @@ -416,29 +416,27 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log fields") CHECK(ec == boost::system::errc::success); CHECK(logValue.is_object()); - CHECK(logValue.as_object().contains("GlobalParams")); - CHECK(logValue.as_object().contains("JournalParams")); - CHECK(logValue.as_object().contains("Metadata")); - CHECK(logValue.as_object().contains("Message")); - - CHECK(logValue.as_object()["GlobalParams"].is_object()); - CHECK(logValue.as_object()["JournalParams"].is_object()); - CHECK(logValue.as_object()["Metadata"].is_object()); - CHECK(logValue.as_object()["Message"].is_string()); - - CHECK(logValue.as_object()["Metadata"].as_object().contains("Function")); - CHECK(logValue.as_object()["Metadata"].as_object().contains("File")); - CHECK(logValue.as_object()["Metadata"].as_object().contains("Line")); - CHECK(logValue.as_object()["Metadata"].as_object().contains("ThreadId")); - CHECK(logValue.as_object()["Metadata"].as_object().contains("Level")); - CHECK(logValue.as_object()["Metadata"].as_object().contains("Time")); - - CHECK(logValue.as_object()["Metadata"].as_object()["Function"].is_string()); - CHECK(logValue.as_object()["Metadata"].as_object()["File"].is_string()); - CHECK(logValue.as_object()["Metadata"].as_object()["Line"].is_number()); + CHECK(logValue.as_object().contains("Glb")); + CHECK(logValue.as_object().contains("Jnl")); + CHECK(logValue.as_object().contains("Mtd")); + CHECK(logValue.as_object().contains("Msg")); + + CHECK(logValue.as_object()["Glb"].is_object()); + CHECK(logValue.as_object()["Jnl"].is_object()); + CHECK(logValue.as_object()["Mtd"].is_object()); + CHECK(logValue.as_object()["Msg"].is_string()); + + CHECK(logValue.as_object()["Mtd"].as_object().contains("Fl")); + CHECK(logValue.as_object()["Mtd"].as_object().contains("Ln")); + CHECK(logValue.as_object()["Mtd"].as_object().contains("ThId")); + CHECK(logValue.as_object()["Mtd"].as_object().contains("Lv")); + CHECK(logValue.as_object()["Mtd"].as_object().contains("Tm")); + + CHECK(logValue.as_object()["Mtd"].as_object()["Fl"].is_string()); + CHECK(logValue.as_object()["Mtd"].as_object()["Ln"].is_number()); CHECK( - logValue.as_object()["Message"].get_string() == + logValue.as_object()["Msg"].get_string() == std::string{"true Test false"}); } @@ -453,8 +451,8 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log levels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["Metadata"] - .as_object()["Level"] + logValue.as_object()["Mtd"] + .as_object()["Lv"] .get_string() == beast::severities::to_string(beast::severities::kTrace)); } @@ -468,8 +466,8 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log levels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["Metadata"] - .as_object()["Level"] + logValue.as_object()["Mtd"] + .as_object()["Lv"] .get_string() == beast::severities::to_string(beast::severities::kDebug)); } @@ -483,8 +481,8 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log levels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["Metadata"] - .as_object()["Level"] + logValue.as_object()["Mtd"] + .as_object()["Lv"] .get_string() == beast::severities::to_string(beast::severities::kInfo)); } @@ -498,8 +496,8 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log levels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["Metadata"] - .as_object()["Level"] + logValue.as_object()["Mtd"] + .as_object()["Lv"] .get_string() == beast::severities::to_string(beast::severities::kWarning)); } @@ -513,8 +511,8 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log levels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["Metadata"] - .as_object()["Level"] + logValue.as_object()["Mtd"] + .as_object()["Lv"] .get_string() == beast::severities::to_string(beast::severities::kError)); } @@ -528,8 +526,8 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log levels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["Metadata"] - .as_object()["Level"] + logValue.as_object()["Mtd"] + .as_object()["Lv"] .get_string() == beast::severities::to_string(beast::severities::kFatal)); } @@ -544,7 +542,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log stream") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["Metadata"].as_object()["Level"].get_string() == + logValue.as_object()["Mtd"].as_object()["Lv"].get_string() == beast::severities::to_string(beast::severities::kError)); } @@ -560,29 +558,29 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log params") auto logValue = boost::json::parse(stream().str(), ec); CHECK(ec == boost::system::errc::success); - CHECK(logValue.as_object()["MessageParams"].is_object()); - CHECK(logValue.as_object()["MessageParams"] + CHECK(logValue.as_object()["Dt"].is_object()); + CHECK(logValue.as_object()["Dt"] .as_object()["Field1"] .is_number()); CHECK( - logValue.as_object()["MessageParams"] + logValue.as_object()["Dt"] .as_object()["Field1"] .get_int64() == 1); - CHECK(logValue.as_object()["MessageParams"] + CHECK(logValue.as_object()["Dt"] .as_object()["Field2"] .is_number()); CHECK( - logValue.as_object()["MessageParams"] + logValue.as_object()["Dt"] .as_object()["Field2"] .get_uint64() == std::numeric_limits::max()); - auto field3Val = logValue.as_object()["MessageParams"] + auto field3Val = logValue.as_object()["Dt"] .as_object()["Field3"] .get_double(); auto difference = std::abs(field3Val - std::numbers::pi); CHECK(difference < 1e-4); - CHECK(logValue.as_object()["Message"].is_string()); + CHECK(logValue.as_object()["Msg"].is_string()); CHECK( - logValue.as_object()["Message"].get_string() == + logValue.as_object()["Msg"].get_string() == std::string{"Test: 1, 18446744073709551615, 3.141592653589793"}); } @@ -597,26 +595,26 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log fields") auto logValue = boost::json::parse(stream().str(), ec); CHECK(ec == boost::system::errc::success); - CHECK(logValue.as_object()["MessageParams"].is_object()); - CHECK(logValue.as_object()["MessageParams"] + CHECK(logValue.as_object()["Dt"].is_object()); + CHECK(logValue.as_object()["Dt"] .as_object()["Field1"] .is_number()); CHECK( - logValue.as_object()["MessageParams"] + logValue.as_object()["Dt"] .as_object()["Field1"] .get_int64() == 1); // UInt64 doesn't fit in Json::Value so it should be converted to a string // NOTE: We should expect it to be an int64 after we make the json library // support in64 and uint64 - CHECK(logValue.as_object()["MessageParams"] + CHECK(logValue.as_object()["Dt"] .as_object()["Field2"] .is_number()); CHECK( - logValue.as_object()["MessageParams"] + logValue.as_object()["Dt"] .as_object()["Field2"] .get_uint64() == std::numeric_limits::max()); - CHECK(logValue.as_object()["Message"].is_string()); - CHECK(logValue.as_object()["Message"].get_string() == "Test"); + CHECK(logValue.as_object()["Msg"].is_string()); + CHECK(logValue.as_object()["Msg"].get_string() == "Test"); } TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test journal attributes") @@ -631,18 +629,18 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test journal attributes") auto logValue = boost::json::parse(stream().str(), ec); CHECK(ec == boost::system::errc::success); - CHECK(logValue.as_object()["JournalParams"] + CHECK(logValue.as_object()["Jnl"] .as_object()["Field1"] .is_string()); CHECK( - logValue.as_object()["JournalParams"] + logValue.as_object()["Jnl"] .as_object()["Field1"] .get_string() == std::string{"Value1"}); - CHECK(logValue.as_object()["JournalParams"] + CHECK(logValue.as_object()["Jnl"] .as_object()["Field2"] .is_number()); CHECK( - logValue.as_object()["JournalParams"] + logValue.as_object()["Jnl"] .as_object()["Field2"] .get_int64() == 2); } @@ -660,25 +658,25 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test journal attributes inheritable") auto logValue = boost::json::parse(stream().str(), ec); CHECK(ec == boost::system::errc::success); - CHECK(logValue.as_object()["JournalParams"] + CHECK(logValue.as_object()["Jnl"] .as_object()["Field1"] .is_string()); CHECK( - logValue.as_object()["JournalParams"] + logValue.as_object()["Jnl"] .as_object()["Field1"] .get_string() == std::string{"Value1"}); - CHECK(logValue.as_object()["JournalParams"] + CHECK(logValue.as_object()["Jnl"] .as_object()["Field3"] .is_string()); CHECK( - logValue.as_object()["JournalParams"] + logValue.as_object()["Jnl"] .as_object()["Field3"] .get_string() == std::string{"Value3"}); - CHECK(logValue.as_object()["JournalParams"] + CHECK(logValue.as_object()["Jnl"] .as_object()["Field2"] .is_number()); CHECK( - logValue.as_object()["JournalParams"] + logValue.as_object()["Jnl"] .as_object()["Field2"] .get_int64() == 2); } @@ -698,18 +696,18 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test copying journal") auto logValue = boost::json::parse(stream().str(), ec); CHECK(ec == boost::system::errc::success); - CHECK(logValue.as_object()["JournalParams"] + CHECK(logValue.as_object()["Jnl"] .as_object()["Field1"] .is_string()); CHECK( - logValue.as_object()["JournalParams"] + logValue.as_object()["Jnl"] .as_object()["Field1"] .get_string() == std::string{"Value1"}); - CHECK(logValue.as_object()["JournalParams"] + CHECK(logValue.as_object()["Jnl"] .as_object()["Field2"] .is_number()); CHECK( - logValue.as_object()["JournalParams"] + logValue.as_object()["Jnl"] .as_object()["Field2"] .get_int64() == 2); } @@ -727,18 +725,18 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test copying journal") auto logValue = boost::json::parse(stream().str(), ec); CHECK(ec == boost::system::errc::success); - CHECK(logValue.as_object()["JournalParams"] + CHECK(logValue.as_object()["Jnl"] .as_object()["Field1"] .is_string()); CHECK( - logValue.as_object()["JournalParams"] + logValue.as_object()["Jnl"] .as_object()["Field1"] .get_string() == std::string{"Value1"}); - CHECK(logValue.as_object()["JournalParams"] + CHECK(logValue.as_object()["Jnl"] .as_object()["Field2"] .is_number()); CHECK( - logValue.as_object()["JournalParams"] + logValue.as_object()["Jnl"] .as_object()["Field2"] .get_int64() == 2); } @@ -759,26 +757,26 @@ TEST_CASE_FIXTURE( auto logValue = boost::json::parse(stream().str(), ec); CHECK(ec == boost::system::errc::success); - CHECK(logValue.as_object()["JournalParams"] + CHECK(logValue.as_object()["Jnl"] .as_object()["Field1"] .is_string()); CHECK( - logValue.as_object()["JournalParams"] + logValue.as_object()["Jnl"] .as_object()["Field1"] .get_string() == std::string{"Value1"}); - CHECK(logValue.as_object()["JournalParams"] + CHECK(logValue.as_object()["Jnl"] .as_object()["Field3"] .is_string()); CHECK( - logValue.as_object()["JournalParams"] + logValue.as_object()["Jnl"] .as_object()["Field3"] .get_string() == std::string{"Value3"}); // Field2 should be overwritten to 0 - CHECK(logValue.as_object()["JournalParams"] + CHECK(logValue.as_object()["Jnl"] .as_object()["Field2"] .is_number()); CHECK( - logValue.as_object()["JournalParams"] + logValue.as_object()["Jnl"] .as_object()["Field2"] .get_int64() == 2); } @@ -801,18 +799,18 @@ TEST_CASE_FIXTURE( auto logValue = boost::json::parse(stream().str(), ec); CHECK(ec == boost::system::errc::success); - CHECK(logValue.as_object()["JournalParams"] + CHECK(logValue.as_object()["Jnl"] .as_object()["Field1"] .is_string()); CHECK( - logValue.as_object()["JournalParams"] + logValue.as_object()["Jnl"] .as_object()["Field1"] .get_string() == std::string{"Value1"}); - CHECK(logValue.as_object()["JournalParams"] + CHECK(logValue.as_object()["Jnl"] .as_object()["Field2"] .is_number()); CHECK( - logValue.as_object()["JournalParams"] + logValue.as_object()["Jnl"] .as_object()["Field2"] .get_int64() == 2); } @@ -835,18 +833,18 @@ TEST_CASE_FIXTURE( auto logValue = boost::json::parse(stream().str(), ec); CHECK(ec == boost::system::errc::success); - CHECK(logValue.as_object()["JournalParams"] + CHECK(logValue.as_object()["Jnl"] .as_object()["Field1"] .is_string()); CHECK( - logValue.as_object()["JournalParams"] + logValue.as_object()["Jnl"] .as_object()["Field1"] .get_string() == std::string{"Value1"}); - CHECK(logValue.as_object()["JournalParams"] + CHECK(logValue.as_object()["Jnl"] .as_object()["Field2"] .is_number()); CHECK( - logValue.as_object()["JournalParams"] + logValue.as_object()["Jnl"] .as_object()["Field2"] .get_int64() == 2); } From 893632d3303e48bba25c2391c89f99ad6464e4ca Mon Sep 17 00:00:00 2001 From: JCW Date: Wed, 10 Sep 2025 11:49:27 +0100 Subject: [PATCH 69/88] Performance test Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 110 +++++++++++++------- src/libxrpl/basics/Log.cpp | 2 +- src/libxrpl/beast/utility/beast_Journal.cpp | 33 +++--- 3 files changed, 91 insertions(+), 54 deletions(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 9a0b5436bb6..2a3e5e3ff68 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -93,6 +93,9 @@ class SimpleJsonWriter { } + std::string& + buffer() { return buffer_; } + void startObject() const { @@ -363,18 +366,18 @@ class Journal std::source_location location, severities::Severity severity, std::string_view moduleName, - std::string_view journalAttributesJson) noexcept; + std::string_view journalAttributes) noexcept; }; private: // Severity level / threshold of a Journal message. using Severity = severities::Severity; - std::string m_name; - std::string m_attributesJson; - static std::string globalLogAttributesJson_; + std::string name_; + std::string attributes_; + static std::string globalLogAttributes_; static std::shared_mutex globalLogAttributesMutex_; - static bool m_jsonLogsEnabled; + static bool jsonLogsEnabled_; static thread_local JsonLogContext currentJsonLogContext_; @@ -618,29 +621,29 @@ class Journal Journal() = delete; Journal(Journal const& other) - : m_name(other.m_name) - , m_attributesJson(other.m_attributesJson) + : name_(other.name_) + , attributes_(other.attributes_) , m_sink(other.m_sink) { } template Journal(Journal const& other, TAttributesFactory&& attributesFactory) - : m_name(other.m_name), m_sink(other.m_sink) + : name_(other.name_), m_sink(other.m_sink) { - std::string buffer{other.m_attributesJson}; + std::string buffer{other.attributes_}; detail::SimpleJsonWriter writer{buffer}; - if (other.m_attributesJson.empty()) + if (other.attributes_.empty() && jsonLogsEnabled_) { writer.startObject(); } attributesFactory(writer); - m_attributesJson = std::move(buffer); + attributes_ = std::move(buffer); } /** Create a journal that writes to the specified sink. */ explicit Journal(Sink& sink, std::string const& name = {}) - : m_name(name), m_sink(&sink) + : name_(name), m_sink(&sink) { } @@ -650,14 +653,14 @@ class Journal Sink& sink, std::string const& name, TAttributesFactory&& attributesFactory) - : m_name(name), m_sink(&sink) + : name_(name), m_sink(&sink) { std::string buffer; buffer.reserve(128); detail::SimpleJsonWriter writer{buffer}; writer.startObject(); attributesFactory(writer); - m_attributesJson = std::move(buffer); + attributes_ = std::move(buffer); } Journal& @@ -667,8 +670,8 @@ class Journal return *this; // LCOV_EXCL_LINE m_sink = other.m_sink; - m_name = other.m_name; - m_attributesJson = other.m_attributesJson; + name_ = other.name_; + attributes_ = other.attributes_; return *this; } @@ -676,8 +679,8 @@ class Journal operator=(Journal&& other) noexcept { m_sink = other.m_sink; - m_name = std::move(other.m_name); - m_attributesJson = std::move(other.m_attributesJson); + name_ = std::move(other.name_); + attributes_ = std::move(other.attributes_); return *this; } @@ -694,8 +697,7 @@ class Journal Severity level, std::source_location location = std::source_location::current()) const { - if (m_jsonLogsEnabled) - initMessageContext(location, level); + initMessageContext(location, level); return Stream(*m_sink, level); } @@ -714,48 +716,42 @@ class Journal Stream trace(std::source_location location = std::source_location::current()) const { - if (m_jsonLogsEnabled) - initMessageContext(location, severities::kTrace); + initMessageContext(location, severities::kTrace); return {*m_sink, severities::kTrace}; } Stream debug(std::source_location location = std::source_location::current()) const { - if (m_jsonLogsEnabled) - initMessageContext(location, severities::kDebug); + initMessageContext(location, severities::kDebug); return {*m_sink, severities::kDebug}; } Stream info(std::source_location location = std::source_location::current()) const { - if (m_jsonLogsEnabled) - initMessageContext(location, severities::kInfo); + initMessageContext(location, severities::kInfo); return {*m_sink, severities::kInfo}; } Stream warn(std::source_location location = std::source_location::current()) const { - if (m_jsonLogsEnabled) - initMessageContext(location, severities::kWarning); + initMessageContext(location, severities::kWarning); return {*m_sink, severities::kWarning}; } Stream error(std::source_location location = std::source_location::current()) const { - if (m_jsonLogsEnabled) - initMessageContext(location, severities::kError); + initMessageContext(location, severities::kError); return {*m_sink, severities::kError}; } Stream fatal(std::source_location location = std::source_location::current()) const { - if (m_jsonLogsEnabled) - initMessageContext(location, severities::kFatal); + initMessageContext(location, severities::kFatal); return {*m_sink, severities::kFatal}; } /** @} */ @@ -764,7 +760,7 @@ class Journal resetGlobalAttributes() { std::unique_lock lock(globalLogAttributesMutex_); - globalLogAttributesJson_.clear(); + globalLogAttributes_.clear(); } template @@ -772,10 +768,10 @@ class Journal addGlobalAttributes(TAttributesFactory&& factory) { std::unique_lock lock(globalLogAttributesMutex_); - globalLogAttributesJson_.reserve(128); - auto isEmpty = globalLogAttributesJson_.empty(); - detail::SimpleJsonWriter writer{globalLogAttributesJson_}; - if (isEmpty) + globalLogAttributes_.reserve(1024); + auto isEmpty = globalLogAttributes_.empty(); + detail::SimpleJsonWriter writer{globalLogAttributes_}; + if (isEmpty && jsonLogsEnabled_) { writer.startObject(); } @@ -905,6 +901,33 @@ concept StreamFormattable = requires(T val) { } -> std::convertible_to; }; +template +void +setTextValue( + beast::detail::SimpleJsonWriter& writer, + char const* name, + T&& value) +{ + using ValueType = std::decay_t; + writer.buffer() += name; + writer.buffer() += ": "; + if constexpr ( + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) + { + writer.buffer() += value; + } + else + { + std::ostringstream oss; + oss << value; + writer.buffer() += value;; + } + writer.buffer() += " "; +} + template void setJsonValue( @@ -1030,7 +1053,7 @@ template std::ostream& operator<<(std::ostream& os, LogParameter const& param) { - if (!beast::Journal::m_jsonLogsEnabled) + if (!beast::Journal::jsonLogsEnabled_) { os << param.value_; return os; @@ -1048,7 +1071,7 @@ template std::ostream& operator<<(std::ostream& os, LogField const& param) { - if (!beast::Journal::m_jsonLogsEnabled) + if (!beast::Journal::jsonLogsEnabled_) return os; beast::Journal::currentJsonLogContext_.startMessageParams(); detail::setJsonValue( @@ -1078,7 +1101,14 @@ template attributes(Pair&&... pairs) { return [&](beast::detail::SimpleJsonWriter& writer) { - (detail::setJsonValue(writer, pairs.first, pairs.second, nullptr), ...); + if (beast::Journal::isStructuredJournalEnabled()) + { + (detail::setJsonValue(writer, pairs.first, pairs.second, nullptr), ...); + } + else + { + (detail::setTextValue(writer, pairs.first, pairs.second), ...); + } }; } diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index 943951dec5b..301fc2bd2d6 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -202,7 +202,7 @@ Logs::write( } // Add to batch buffer for file output - { + if (false) { std::lock_guard lock(batchMutex_); // Console output still immediate for responsiveness diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index 89de4baaf9d..b5163bad935 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -110,9 +110,9 @@ fastTimestampToString(std::int64_t milliseconds_since_epoch) } // anonymous namespace -std::string Journal::globalLogAttributesJson_; +std::string Journal::globalLogAttributes_; std::shared_mutex Journal::globalLogAttributesMutex_; -bool Journal::m_jsonLogsEnabled = false; +bool Journal::jsonLogsEnabled_ = false; thread_local Journal::JsonLogContext Journal::currentJsonLogContext_{}; //------------------------------------------------------------------------------ @@ -208,7 +208,7 @@ Journal::JsonLogContext::reset( std::source_location location, severities::Severity severity, std::string_view moduleName, - std::string_view journalAttributesJson) noexcept + std::string_view journalAttributes) noexcept { struct ThreadIdStringInitializer { @@ -224,21 +224,26 @@ Journal::JsonLogContext::reset( buffer_.clear(); + if (!jsonLogsEnabled_) + { + return; + } + writer().startObject(); - if (!journalAttributesJson.empty()) + if (!journalAttributes.empty()) { writer().writeKey("Jnl"); - writer().writeRaw(journalAttributesJson); + writer().writeRaw(journalAttributes); writer().endObject(); } { std::shared_lock lock(globalLogAttributesMutex_); - if (!globalLogAttributesJson_.empty()) + if (!globalLogAttributes_.empty()) { writer().writeKey("Glb"); - writer().writeRaw(globalLogAttributesJson_); + writer().writeRaw(globalLogAttributes_); writer().endObject(); } } @@ -283,15 +288,17 @@ Journal::initMessageContext( std::source_location location, severities::Severity severity) const { - currentJsonLogContext_.reset(location, severity, m_name, m_attributesJson); + currentJsonLogContext_.reset(location, severity, name_, attributes_); } std::string_view Journal::formatLog(std::string const& message) { - if (!m_jsonLogsEnabled) + if (!jsonLogsEnabled_) { - return message; + currentJsonLogContext_.writer().buffer() += " "; + currentJsonLogContext_.writer().buffer() += message; + return currentJsonLogContext_.writer().buffer(); } auto& writer = currentJsonLogContext_.writer(); @@ -309,20 +316,20 @@ Journal::formatLog(std::string const& message) void Journal::enableStructuredJournal() { - m_jsonLogsEnabled = true; + jsonLogsEnabled_ = true; } void Journal::disableStructuredJournal() { - m_jsonLogsEnabled = false; + jsonLogsEnabled_ = false; resetGlobalAttributes(); } bool Journal::isStructuredJournalEnabled() { - return m_jsonLogsEnabled; + return jsonLogsEnabled_; } Journal::Sink::Sink(Severity thresh, bool console) From c4047690e2dff60c4149605e688f864dbd813d1f Mon Sep 17 00:00:00 2001 From: JCW Date: Wed, 10 Sep 2025 12:02:00 +0100 Subject: [PATCH 70/88] Fix test cases Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 5 ++++- src/tests/libxrpl/basics/log.cpp | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 2a3e5e3ff68..42a4ed5e8b7 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -658,7 +658,10 @@ class Journal std::string buffer; buffer.reserve(128); detail::SimpleJsonWriter writer{buffer}; - writer.startObject(); + if (jsonLogsEnabled_) + { + writer.startObject(); + } attributesFactory(writer); attributes_ = std::move(buffer); } diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index d550c7f0401..9d12202ea66 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -375,9 +375,11 @@ class JsonLogStreamFixture public: JsonLogStreamFixture() : sink_(beast::severities::kAll, logStream_) - , j_(sink_, "Test", log::attributes(log::attr("Field1", "Value1"))) + , j_(beast::Journal::getNullSink()) { + beast::Journal::resetGlobalAttributes(); beast::Journal::enableStructuredJournal(); + j_ = beast::Journal{sink_, "Test", log::attributes(log::attr("Field1", "Value1"))}; } ~JsonLogStreamFixture() From a854a78107a4db1ad06d631de89a15bd9b7e7d68 Mon Sep 17 00:00:00 2001 From: JCW Date: Wed, 10 Sep 2025 13:38:42 +0100 Subject: [PATCH 71/88] Fix test error Signed-off-by: JCW --- src/libxrpl/beast/utility/beast_Journal.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index b5163bad935..3421fd7a6d3 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -296,7 +296,6 @@ Journal::formatLog(std::string const& message) { if (!jsonLogsEnabled_) { - currentJsonLogContext_.writer().buffer() += " "; currentJsonLogContext_.writer().buffer() += message; return currentJsonLogContext_.writer().buffer(); } From d2f01eb755489502adfa99fa3d1606df28a393ef Mon Sep 17 00:00:00 2001 From: JCW Date: Wed, 10 Sep 2025 17:08:59 +0100 Subject: [PATCH 72/88] Performance test Signed-off-by: JCW --- src/libxrpl/basics/Log.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index 301fc2bd2d6..2b17f23671b 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -202,7 +202,7 @@ Logs::write( } // Add to batch buffer for file output - if (false) { + { std::lock_guard lock(batchMutex_); // Console output still immediate for responsiveness @@ -221,6 +221,8 @@ Logs::write( std::copy(result.begin(), result.end(), writeBuffer_.begin()); writeBuffer_[result.size()] = '\n'; + return; + // Update spans: expand read buffer, shrink write buffer size_t totalUsed = readBuffer_.size() + logSize; readBuffer_ = std::span(batchBuffer_.data(), totalUsed); From 294dae5766cfd9254a74b9e507679ebe47cf8614 Mon Sep 17 00:00:00 2001 From: JCW Date: Wed, 10 Sep 2025 21:23:07 +0100 Subject: [PATCH 73/88] Performance test Signed-off-by: JCW --- src/libxrpl/basics/Log.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index 2b17f23671b..a9b55f8f426 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -206,8 +206,8 @@ Logs::write( std::lock_guard lock(batchMutex_); // Console output still immediate for responsiveness - if (!silent_) - std::cerr << result << '\n'; + // if (!silent_) + // std::cerr << result << '\n'; size_t logSize = result.size() + 1; // +1 for newline @@ -218,7 +218,7 @@ Logs::write( } // Copy log into write buffer - std::copy(result.begin(), result.end(), writeBuffer_.begin()); + // std::copy(result.begin(), result.end(), writeBuffer_.begin()); writeBuffer_[result.size()] = '\n'; return; From a4498f084eae7f49476e48068d2f48c8a3cc5d69 Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 11 Sep 2025 07:59:17 +0100 Subject: [PATCH 74/88] Performance test Signed-off-by: JCW --- src/libxrpl/basics/Log.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index a9b55f8f426..7b548fc05af 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -203,7 +203,7 @@ Logs::write( // Add to batch buffer for file output { - std::lock_guard lock(batchMutex_); + // std::lock_guard lock(batchMutex_); // Console output still immediate for responsiveness // if (!silent_) @@ -218,7 +218,7 @@ Logs::write( } // Copy log into write buffer - // std::copy(result.begin(), result.end(), writeBuffer_.begin()); + std::copy(result.begin(), result.end(), writeBuffer_.begin()); writeBuffer_[result.size()] = '\n'; return; From 211d90dadd60f14abb220cf15a73a47c1bddb227 Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 11 Sep 2025 08:06:03 +0100 Subject: [PATCH 75/88] Performance test Signed-off-by: JCW --- include/xrpl/basics/Log.h | 2 +- src/libxrpl/basics/Log.cpp | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/xrpl/basics/Log.h b/include/xrpl/basics/Log.h index dae3fa14afd..e2112d2f7ff 100644 --- a/include/xrpl/basics/Log.h +++ b/include/xrpl/basics/Log.h @@ -155,7 +155,7 @@ class Logs mutable std::mutex batchMutex_; static constexpr size_t BATCH_BUFFER_SIZE = 64 * 1024; // 64KB buffer std::array batchBuffer_{}; - std::span writeBuffer_; // Points to available write space + static thread_local std::span writeBuffer_; // Points to available write space std::span readBuffer_; // Points to data ready to flush std::chrono::steady_clock::time_point lastFlush_ = std::chrono::steady_clock::now(); diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index 7b548fc05af..bd5cbd8bf5d 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -37,6 +37,7 @@ #include namespace ripple { +thread_local std::span Logs::writeBuffer_{}; namespace { constexpr auto FLUSH_INTERVAL = @@ -129,8 +130,8 @@ Logs::File::write(std::string_view text) Logs::Logs(beast::severities::Severity thresh) : thresh_(thresh) // default severity - , writeBuffer_( - batchBuffer_) // Initially, entire buffer is available for writing + // , writeBuffer_( + // batchBuffer_) // Initially, entire buffer is available for writing , readBuffer_(batchBuffer_.data(), 0) // No data ready to flush initially { } From 45a4f44dc1934d4e16971452ec7638b11d5014e8 Mon Sep 17 00:00:00 2001 From: JCW Date: Thu, 11 Sep 2025 08:26:57 +0100 Subject: [PATCH 76/88] Performance test Signed-off-by: JCW --- include/xrpl/basics/Log.h | 4 +++- src/libxrpl/basics/Log.cpp | 5 ++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/include/xrpl/basics/Log.h b/include/xrpl/basics/Log.h index e2112d2f7ff..7c9e566ed43 100644 --- a/include/xrpl/basics/Log.h +++ b/include/xrpl/basics/Log.h @@ -153,10 +153,12 @@ class Logs // Batching members mutable std::mutex batchMutex_; +public: static constexpr size_t BATCH_BUFFER_SIZE = 64 * 1024; // 64KB buffer std::array batchBuffer_{}; - static thread_local std::span writeBuffer_; // Points to available write space + std::span writeBuffer_; // Points to available write space std::span readBuffer_; // Points to data ready to flush +private: std::chrono::steady_clock::time_point lastFlush_ = std::chrono::steady_clock::now(); diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index bd5cbd8bf5d..7b548fc05af 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -37,7 +37,6 @@ #include namespace ripple { -thread_local std::span Logs::writeBuffer_{}; namespace { constexpr auto FLUSH_INTERVAL = @@ -130,8 +129,8 @@ Logs::File::write(std::string_view text) Logs::Logs(beast::severities::Severity thresh) : thresh_(thresh) // default severity - // , writeBuffer_( - // batchBuffer_) // Initially, entire buffer is available for writing + , writeBuffer_( + batchBuffer_) // Initially, entire buffer is available for writing , readBuffer_(batchBuffer_.data(), 0) // No data ready to flush initially { } From 4feaa7b27942ac021c1cdb528fab5be238a3ef2d Mon Sep 17 00:00:00 2001 From: JCW Date: Fri, 12 Sep 2025 14:28:26 +0100 Subject: [PATCH 77/88] WIP Signed-off-by: JCW --- include/xrpl/basics/Log.h | 24 ++- include/xrpl/beast/utility/Journal.h | 197 +++++++++++++++++++- include/xrpl/beast/utility/WrappedSink.h | 8 +- src/libxrpl/basics/Log.cpp | 90 +++++++-- src/libxrpl/beast/utility/beast_Journal.cpp | 22 ++- src/test/beast/beast_Journal_test.cpp | 4 +- src/test/csf/Sim.h | 4 +- src/test/jtx/CaptureLogs.h | 4 +- src/test/jtx/CheckMessageLogs.h | 6 +- src/test/server/Server_test.cpp | 4 +- src/test/unit_test/SuiteJournal.h | 17 +- src/tests/libxrpl/basics/log.cpp | 8 +- src/xrpld/app/consensus/RCLConsensus.cpp | 9 +- 13 files changed, 330 insertions(+), 67 deletions(-) diff --git a/include/xrpl/basics/Log.h b/include/xrpl/basics/Log.h index 7c9e566ed43..f24c89c3d6a 100644 --- a/include/xrpl/basics/Log.h +++ b/include/xrpl/basics/Log.h @@ -27,12 +27,15 @@ #include #include +#include #include +#include #include #include #include #include #include +#include #include namespace ripple { @@ -71,10 +74,10 @@ class Logs operator=(Sink const&) = delete; void - write(beast::severities::Severity level, std::string_view text) override; + write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override; void - writeAlways(beast::severities::Severity level, std::string_view text) + writeAlways(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override; }; @@ -132,12 +135,12 @@ class Logs Does nothing if there is no associated system file. */ void - write(std::string_view str); + write(std::string&& str); /** @} */ private: - std::unique_ptr m_stream; + std::optional m_stream; boost::filesystem::path m_path; }; @@ -153,11 +156,18 @@ class Logs // Batching members mutable std::mutex batchMutex_; -public: + beast::lockfree::queue messages_; static constexpr size_t BATCH_BUFFER_SIZE = 64 * 1024; // 64KB buffer std::array batchBuffer_{}; std::span writeBuffer_; // Points to available write space std::span readBuffer_; // Points to data ready to flush + + // Log thread members + std::thread logThread_; + std::atomic stopLogThread_; + std::mutex logMutex_; + std::condition_variable logCondition_; + private: std::chrono::steady_clock::time_point lastFlush_ = std::chrono::steady_clock::now(); @@ -208,6 +218,7 @@ class Logs beast::severities::Severity level, std::string const& partition, std::string_view text, + beast::Journal::MessagePoolNode owner, bool console); std::string @@ -261,6 +272,9 @@ class Logs void flushBatchUnsafe(); + + void + logThreadWorker(); }; // Wraps a Journal::Stream to skip evaluation of diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 42a4ed5e8b7..1c30644de48 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -22,6 +22,7 @@ #include +#include #include #include #include @@ -34,6 +35,164 @@ #include #include +namespace beast { + +class StringBufferPool { +public: + // ----- Empty index marker ----- + static constexpr std::uint32_t kEmptyIdx = std::numeric_limits::max(); + + // ----- Single-word CAS target: {tag | idx} with pack/unpack ----- + struct Head { + std::uint32_t tag; + std::uint32_t idx; // kEmptyIdx means empty + + static std::uint64_t pack(Head h) noexcept { + return (std::uint64_t(h.tag) << 32) | h.idx; + } + static Head unpack(std::uint64_t v) noexcept { + return Head{ std::uint32_t(v >> 32), std::uint32_t(v) }; + } + }; + + // ----- Internal node ----- + struct Node { + std::uint32_t next_idx{kEmptyIdx}; + std::uint32_t self_idx{kEmptyIdx}; + std::string buf{}; + }; + static_assert(std::is_standard_layout_v, "Node must be standard layout"); + + // ----- User-facing move-only RAII handle ----- + class Handle { + public: + Handle() = default; + Handle(Handle&& other) noexcept + : owner_(other.owner_), node_(other.node_) { + other.owner_ = nullptr; other.node_ = nullptr; + } + Handle& operator=(Handle&& other) noexcept { + if (this != &other) { + // Return current if still held + if (owner_ && node_) owner_->give_back(std::move(*this)); + owner_ = other.owner_; + node_ = other.node_; + other.owner_ = nullptr; + other.node_ = nullptr; + } + return *this; + } + + Handle(const Handle&) = delete; + Handle& operator=(const Handle&) = delete; + + ~Handle() noexcept { + if (owner_ && node_) owner_->give_back(std::move(*this)); + } + + bool valid() const noexcept { return node_ != nullptr; } + std::string& string() noexcept { return node_->buf; } + const std::string& string() const noexcept { return node_->buf; } + + private: + friend class StringBufferPool; + Handle(StringBufferPool* owner, Node* n) : owner_(owner), node_(n) {} + + StringBufferPool* owner_ = nullptr; + Node* node_ = nullptr; + }; + + explicit StringBufferPool(std::uint32_t grow_by = 20) + : grow_by_(grow_by), head_(Head::pack({0, kEmptyIdx})) {} + + // Rent a buffer; grows on demand. Returns move-only RAII handle. + Handle rent() { + for (;;) { + std::uint64_t old64 = head_.load(std::memory_order_acquire); + Head old = Head::unpack(old64); + if (old.idx == kEmptyIdx) { grow_(); continue; } // rare slow path + + Node& n = nodes_[old.idx]; + std::uint32_t next = n.next_idx; + + Head neu{ std::uint32_t(old.tag + 1), next }; + if (head_.compare_exchange_weak(old64, Head::pack(neu), + std::memory_order_acq_rel, + std::memory_order_acquire)) { + return {this, &n}; + } + } + } + +private: + // Only the pool/handle can call this + void give_back(Handle&& h) noexcept { + Node* node = h.node_; + if (!node) return; // already invalid + const std::uint32_t idx = node->self_idx; + + node->buf.clear(); + + for (;;) { + std::uint64_t old64 = head_.load(std::memory_order_acquire); + Head old = Head::unpack(old64); + + node->next_idx = old.idx; + + Head neu{ std::uint32_t(old.tag + 1), idx }; + if (head_.compare_exchange_weak(old64, Head::pack(neu), + std::memory_order_acq_rel, + std::memory_order_acquire)) { + // Invalidate handle (prevents double return) + h.owner_ = nullptr; + h.node_ = nullptr; + return; + } + } + } + + void grow_() { + if (Head::unpack(head_.load(std::memory_order_acquire)).idx != kEmptyIdx) return; + std::scoped_lock lk(grow_mu_); + if (Head::unpack(head_.load(std::memory_order_acquire)).idx != kEmptyIdx) return; + + const std::uint32_t base = static_cast(nodes_.size()); + nodes_.resize(base + grow_by_); // indices [base .. base+grow_by_-1] + + // Init nodes and local chain + for (std::uint32_t i = 0; i < grow_by_; ++i) { + std::uint32_t idx = base + i; + Node& n = nodes_[idx]; + n.self_idx = idx; + n.next_idx = (i + 1 < grow_by_) ? (idx + 1) : kEmptyIdx; + } + + // Splice chain onto global head: [base .. base+grow_by_-1] + const std::uint32_t chain_head = base; + const std::uint32_t chain_tail = base + grow_by_ - 1; + + for (;;) { + std::uint64_t old64 = head_.load(std::memory_order_acquire); + Head old = Head::unpack(old64); + + nodes_[chain_tail].next_idx = old.idx; // tail -> old head + Head neu{ std::uint32_t(old.tag + 1), chain_head }; + + if (head_.compare_exchange_weak(old64, Head::pack(neu), + std::memory_order_acq_rel, + std::memory_order_acquire)) { + break; + } + } + } + + const std::uint32_t grow_by_; + std::atomic head_; // single 64-bit CAS (Head packed) + std::mutex grow_mu_; // only during growth + std::deque nodes_; // stable storage for nodes/strings +}; +} // namespace beast + namespace ripple::log { template class LogParameter @@ -181,10 +340,10 @@ class SimpleJsonWriter buffer_.append(str); } - [[nodiscard]] std::string_view + void finish() { - return std::string_view{buffer_.c_str(), buffer_.size() - 1}; + buffer_.pop_back(); } private: @@ -323,18 +482,25 @@ class Journal class Sink; + using MessagePoolNode = lockfree::queue::Node*; + class JsonLogContext { - std::string buffer_; + MessagePoolNode messageBuffer_; detail::SimpleJsonWriter messageParamsWriter_; bool hasMessageParams_ = false; public: - JsonLogContext() : messageParamsWriter_(buffer_) + explicit JsonLogContext() + : messageBuffer_(rentFromPool()) + , messageParamsWriter_(messageBuffer_->data) { - buffer_.reserve(1024 * 5); + messageBuffer_->data.reserve(1024 * 5); } + MessagePoolNode + messageBuffer() { return messageBuffer_; } + void startMessageParams() { @@ -379,6 +545,7 @@ class Journal static std::shared_mutex globalLogAttributesMutex_; static bool jsonLogsEnabled_; + static lockfree::queue messagePool_; static thread_local JsonLogContext currentJsonLogContext_; // Invariant: m_sink always points to a valid Sink @@ -389,12 +556,26 @@ class Journal std::source_location location, severities::Severity severity) const; - static std::string_view + static MessagePoolNode formatLog(std::string const& message); public: //-------------------------------------------------------------------------- + static MessagePoolNode + rentFromPool() + { + auto node = messagePool_.pop(); + if (!node) + { + node = new lockfree::queue::Node(); + } + return node; + } + + static void + returnMessageNode(MessagePoolNode node) { messagePool_.push(node); } + static void enableStructuredJournal(); @@ -444,7 +625,7 @@ class Journal level is below the current threshold(). */ virtual void - write(Severity level, std::string_view text) = 0; + write(Severity level, std::string_view text, MessagePoolNode owner = nullptr) = 0; /** Bypass filter and write text to the sink at the specified severity. * Always write the message, but maintain the same formatting as if @@ -454,7 +635,7 @@ class Journal * @param text Text to write to sink. */ virtual void - writeAlways(Severity level, std::string_view text) = 0; + writeAlways(Severity level, std::string_view text, MessagePoolNode owner = nullptr) = 0; private: Severity thresh_; diff --git a/include/xrpl/beast/utility/WrappedSink.h b/include/xrpl/beast/utility/WrappedSink.h index e1343dc9e69..6024e6b7972 100644 --- a/include/xrpl/beast/utility/WrappedSink.h +++ b/include/xrpl/beast/utility/WrappedSink.h @@ -88,17 +88,17 @@ class WrappedSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, std::string_view text) override + write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override { using beast::Journal; - sink_.write(level, prefix_ + std::string{text}); + sink_.write(level, prefix_ + std::string(text), owner); } void - writeAlways(severities::Severity level, std::string_view text) override + writeAlways(severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override { using beast::Journal; - sink_.writeAlways(level, prefix_ + std::string{text}); + sink_.writeAlways(level, prefix_ + std::string(text), owner); } }; diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index 7b548fc05af..07fdffe6827 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -43,6 +43,7 @@ constexpr auto FLUSH_INTERVAL = std::chrono::milliseconds(10); // Max delay before flush } + Logs::Sink::Sink( std::string const& partition, beast::severities::Severity thresh, @@ -52,18 +53,18 @@ Logs::Sink::Sink( } void -Logs::Sink::write(beast::severities::Severity level, std::string_view text) +Logs::Sink::write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner) { if (level < threshold()) return; - logs_.write(level, partition_, text, console()); + logs_.write(level, partition_, text, owner, console()); } void -Logs::Sink::writeAlways(beast::severities::Severity level, std::string_view text) +Logs::Sink::writeAlways(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner) { - logs_.write(level, partition_, text, console()); + logs_.write(level, partition_, text, owner, console()); } //------------------------------------------------------------------------------ @@ -75,7 +76,7 @@ Logs::File::File() : m_stream(nullptr) bool Logs::File::isOpen() const noexcept { - return m_stream != nullptr; + return m_stream.has_value(); } bool @@ -86,10 +87,9 @@ Logs::File::open(boost::filesystem::path const& path) bool wasOpened = false; // VFALCO TODO Make this work with Unicode file paths - std::unique_ptr stream( - new std::ofstream(path.c_str(), std::fstream::app)); + std::ofstream stream(path.c_str(), std::fstream::app); - if (stream->good()) + if (stream.good()) { m_path = path; @@ -115,13 +115,13 @@ Logs::File::closeAndReopen() void Logs::File::close() { - m_stream = nullptr; + m_stream.reset(); } void -Logs::File::write(std::string_view text) +Logs::File::write(std::string&& text) { - if (m_stream != nullptr) + if (m_stream.has_value()) m_stream->write(text.data(), text.size()); } @@ -132,11 +132,23 @@ Logs::Logs(beast::severities::Severity thresh) , writeBuffer_( batchBuffer_) // Initially, entire buffer is available for writing , readBuffer_(batchBuffer_.data(), 0) // No data ready to flush initially + , stopLogThread_(false) { + logThread_ = std::thread(&Logs::logThreadWorker, this); } Logs::~Logs() { + // Signal log thread to stop and wait for it to finish + { + std::lock_guard lock(logMutex_); + stopLogThread_ = true; + } + logCondition_.notify_all(); + + if (logThread_.joinable()) + logThread_.join(); + flushBatch(); // Ensure all logs are written on shutdown } @@ -191,6 +203,7 @@ Logs::write( beast::severities::Severity level, std::string const& partition, std::string_view text, + beast::Journal::MessagePoolNode owner, bool console) { std::string s; @@ -201,8 +214,18 @@ Logs::write( result = s; } + // if (!silent_) + // std::cerr << s << '\n'; + + // Get a node from the pool or create a new one + if (!owner) return; + messages_.push(owner); + + // Signal log thread that new messages are available + logCondition_.notify_one(); + // Add to batch buffer for file output - { + if (0) { // std::lock_guard lock(batchMutex_); // Console output still immediate for responsiveness @@ -258,13 +281,54 @@ Logs::flushBatchUnsafe() return; // Write the read buffer contents to file in one system call - file_.write(std::string_view{readBuffer_.data(), readBuffer_.size()}); + // file_.write(std::string_view{readBuffer_.data(), readBuffer_.size()}); // Reset spans: entire buffer available for writing, nothing to read writeBuffer_ = std::span(batchBuffer_); readBuffer_ = std::span(batchBuffer_.data(), 0); } +void +Logs::logThreadWorker() +{ + beast::lockfree::queue::Node* node; + + while (!stopLogThread_) + { + std::unique_lock lock(logMutex_); + + // Wait for messages or stop signal + logCondition_.wait(lock, [this] { + return stopLogThread_ || !messages_.empty(); + }); + + // Process all available messages + while ((node = messages_.pop())) + { + // Write to file + file_.write(std::move(node->data)); + + // Also write to console if not silent + if (!silent_) + std::cerr << node->data << '\n'; + + // Return node to pool for reuse + beast::Journal::returnMessageNode(node); + } + } + + // Process any remaining messages on shutdown + while ((node = messages_.pop())) + { + file_.write(std::move(node->data)); + if (!silent_) + std::cerr << node->data << '\n'; + + // Return node to pool for reuse + beast::Journal::returnMessageNode(node); + } +} + std::string Logs::rotate() { diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index 3421fd7a6d3..b7a7c7ef187 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -113,6 +113,7 @@ fastTimestampToString(std::int64_t milliseconds_since_epoch) std::string Journal::globalLogAttributes_; std::shared_mutex Journal::globalLogAttributesMutex_; bool Journal::jsonLogsEnabled_ = false; +lockfree::queue Journal::messagePool_{}; thread_local Journal::JsonLogContext Journal::currentJsonLogContext_{}; //------------------------------------------------------------------------------ @@ -156,12 +157,12 @@ class NullJournalSink : public Journal::Sink } void - write(severities::Severity, std::string_view) override + write(severities::Severity, std::string_view, Journal::MessagePoolNode = nullptr) override { } void - writeAlways(severities::Severity, std::string_view) override + writeAlways(severities::Severity, std::string_view, Journal::MessagePoolNode = nullptr) override { } }; @@ -222,7 +223,7 @@ Journal::JsonLogContext::reset( }; thread_local ThreadIdStringInitializer const threadId; - buffer_.clear(); + messageBuffer_->data.clear(); if (!jsonLogsEnabled_) { @@ -291,13 +292,13 @@ Journal::initMessageContext( currentJsonLogContext_.reset(location, severity, name_, attributes_); } -std::string_view +Journal::MessagePoolNode Journal::formatLog(std::string const& message) { if (!jsonLogsEnabled_) { currentJsonLogContext_.writer().buffer() += message; - return currentJsonLogContext_.writer().buffer(); + return currentJsonLogContext_.messageBuffer(); } auto& writer = currentJsonLogContext_.writer(); @@ -309,7 +310,9 @@ Journal::formatLog(std::string const& message) writer.endObject(); - return writer.finish(); + writer.finish(); + + return currentJsonLogContext_.messageBuffer(); } void @@ -391,9 +394,10 @@ Journal::ScopedStream::~ScopedStream() if (!s.empty()) { if (s == "\n") - m_sink.write(m_level, formatLog("")); - else - m_sink.write(m_level, formatLog(s)); + s = ""; + + auto messageHandle = formatLog(s); + m_sink.write(m_level, messageHandle->data, messageHandle); } } diff --git a/src/test/beast/beast_Journal_test.cpp b/src/test/beast/beast_Journal_test.cpp index b1379a34563..e50d47b2e86 100644 --- a/src/test/beast/beast_Journal_test.cpp +++ b/src/test/beast/beast_Journal_test.cpp @@ -48,14 +48,14 @@ class Journal_test : public unit_test::suite } void - write(severities::Severity level, std::string_view) override + write(severities::Severity level, std::string_view, beast::Journal::MessagePoolNode = nullptr) override { if (level >= threshold()) ++m_count; } void - writeAlways(severities::Severity level, std::string_view) override + writeAlways(severities::Severity level, std::string_view, beast::Journal::MessagePoolNode = nullptr) override { ++m_count; } diff --git a/src/test/csf/Sim.h b/src/test/csf/Sim.h index 313f8abd8ff..7a606b40bf9 100644 --- a/src/test/csf/Sim.h +++ b/src/test/csf/Sim.h @@ -49,7 +49,7 @@ class BasicSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, std::string_view text) override + write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override { if (level < threshold()) return; @@ -59,7 +59,7 @@ class BasicSink : public beast::Journal::Sink } void - writeAlways(beast::severities::Severity level, std::string_view text) override + writeAlways(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override { std::cout << clock_.now().time_since_epoch().count() << " " << text << std::endl; diff --git a/src/test/jtx/CaptureLogs.h b/src/test/jtx/CaptureLogs.h index 4fffae130ef..2abcc6bd251 100644 --- a/src/test/jtx/CaptureLogs.h +++ b/src/test/jtx/CaptureLogs.h @@ -57,14 +57,14 @@ class CaptureLogs : public Logs } void - write(beast::severities::Severity level, std::string_view text) override + write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override { std::lock_guard lock(strmMutex_); strm_ << text; } void - writeAlways(beast::severities::Severity level, std::string_view text) + writeAlways(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override { std::lock_guard lock(strmMutex_); diff --git a/src/test/jtx/CheckMessageLogs.h b/src/test/jtx/CheckMessageLogs.h index 6b950a82eeb..18ae98b196e 100644 --- a/src/test/jtx/CheckMessageLogs.h +++ b/src/test/jtx/CheckMessageLogs.h @@ -45,17 +45,17 @@ class CheckMessageLogs : public Logs } void - write(beast::severities::Severity level, std::string_view text) override + write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override { if (text.find(owner_.msg_) != std::string::npos) *owner_.pFound_ = true; } void - writeAlways(beast::severities::Severity level, std::string_view text) + writeAlways(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override { - write(level, std::move(text)); + write(level, text, owner); } }; diff --git a/src/test/server/Server_test.cpp b/src/test/server/Server_test.cpp index 1c5aa67992c..3ebed8b699f 100644 --- a/src/test/server/Server_test.cpp +++ b/src/test/server/Server_test.cpp @@ -92,7 +92,7 @@ class Server_test : public beast::unit_test::suite } void - write(beast::severities::Severity level, std::string_view text) override + write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override { if (level < threshold()) return; @@ -101,7 +101,7 @@ class Server_test : public beast::unit_test::suite } void - writeAlways(beast::severities::Severity level, std::string_view text) + writeAlways(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override { suite_.log << text << std::endl; diff --git a/src/test/unit_test/SuiteJournal.h b/src/test/unit_test/SuiteJournal.h index e58e435e6f1..fc0f85d69a4 100644 --- a/src/test/unit_test/SuiteJournal.h +++ b/src/test/unit_test/SuiteJournal.h @@ -49,24 +49,25 @@ class SuiteJournalSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, std::string_view text) override; + write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override; void - writeAlways(beast::severities::Severity level, std::string_view text) override; + writeAlways(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override; }; inline void -SuiteJournalSink::write(beast::severities::Severity level, std::string_view text) +SuiteJournalSink::write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner) { // Only write the string if the level at least equals the threshold. if (level >= threshold()) - writeAlways(level, std::move(text)); + writeAlways(level, text, owner); } inline void SuiteJournalSink::writeAlways( beast::severities::Severity level, - std::string_view text) + std::string_view text, + beast::Journal::MessagePoolNode owner) { using namespace beast::severities; @@ -134,15 +135,15 @@ class StreamSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, std::string_view text) override + write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override { if (level < threshold()) return; - writeAlways(level, std::move(text)); + writeAlways(level, text, owner); } inline void - writeAlways(beast::severities::Severity level, std::string_view text) override + writeAlways(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode = nullptr) override { strm_ << text << std::endl; } diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index 9d12202ea66..10c2bb4c891 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -53,13 +53,13 @@ class MockLogs : public Logs operator=(Sink const&) = delete; void - write(beast::severities::Severity level, std::string_view text) override + write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override { logs_.write(level, partition_, text, false); } void - writeAlways(beast::severities::Severity level, std::string_view text) + writeAlways(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override { logs_.write(level, partition_, text, false); @@ -358,13 +358,13 @@ class MockSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, std::string_view text) override + write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override { strm_ << text; } void - writeAlways(beast::severities::Severity level, std::string_view text) override + writeAlways(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override { strm_ << text; } diff --git a/src/xrpld/app/consensus/RCLConsensus.cpp b/src/xrpld/app/consensus/RCLConsensus.cpp index 4e13edc649a..8dfd48c761b 100644 --- a/src/xrpld/app/consensus/RCLConsensus.cpp +++ b/src/xrpld/app/consensus/RCLConsensus.cpp @@ -1134,17 +1134,16 @@ RclConsensusLogger::~RclConsensusLogger() if (beast::Journal::isStructuredJournalEnabled()) { - thread_local std::string buffer; - buffer.reserve(1024); - buffer.clear(); - beast::detail::SimpleJsonWriter writer{buffer}; + auto node = beast::Journal::rentFromPool(); + beast::detail::SimpleJsonWriter writer{node->data}; writer.startObject(); writer.writeKey("Msg"); writer.writeString(outSs.str()); writer.writeKey("Tm"); writer.writeString(to_string(std::chrono::system_clock::now())); writer.endObject(); - j_.sink().writeAlways(beast::severities::kInfo, writer.finish()); + writer.finish(); + j_.sink().writeAlways(beast::severities::kInfo, writer.buffer(), node); } else { From acafed737620548f0a4d93c0ebd5818375101d16 Mon Sep 17 00:00:00 2001 From: JCW Date: Fri, 12 Sep 2025 18:38:19 +0100 Subject: [PATCH 78/88] Performance test Signed-off-by: JCW --- include/xrpl/basics/Log.h | 27 +- include/xrpl/beast/utility/Journal.h | 383 +++++++++----------- include/xrpl/beast/utility/WrappedSink.h | 10 +- src/libxrpl/basics/Log.cpp | 67 ++-- src/libxrpl/beast/utility/beast_Journal.cpp | 42 ++- src/test/beast/beast_Journal_test.cpp | 4 +- src/test/csf/Sim.h | 8 +- src/test/jtx/CaptureLogs.h | 8 +- src/test/jtx/CheckMessageLogs.h | 8 +- src/test/server/Server_test.cpp | 8 +- src/test/unit_test/SuiteJournal.h | 21 +- src/tests/libxrpl/basics/log.cpp | 40 +- src/xrpld/app/consensus/RCLConsensus.cpp | 8 +- 13 files changed, 300 insertions(+), 334 deletions(-) diff --git a/include/xrpl/basics/Log.h b/include/xrpl/basics/Log.h index f24c89c3d6a..cabdbc70d50 100644 --- a/include/xrpl/basics/Log.h +++ b/include/xrpl/basics/Log.h @@ -26,6 +26,8 @@ #include #include +#include + #include #include #include @@ -74,10 +76,10 @@ class Logs operator=(Sink const&) = delete; void - write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override; + write(beast::severities::Severity level, beast::Journal::StringBuffer text) override; void - writeAlways(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) + writeAlways(beast::severities::Severity level, beast::Journal::StringBuffer text) override; }; @@ -135,7 +137,7 @@ class Logs Does nothing if there is no associated system file. */ void - write(std::string&& str); + write(std::string const& str); /** @} */ @@ -156,7 +158,7 @@ class Logs // Batching members mutable std::mutex batchMutex_; - beast::lockfree::queue messages_; + boost::lockfree::queue> messages_; static constexpr size_t BATCH_BUFFER_SIZE = 64 * 1024; // 64KB buffer std::array batchBuffer_{}; std::span writeBuffer_; // Points to available write space @@ -217,8 +219,7 @@ class Logs write( beast::severities::Severity level, std::string const& partition, - std::string_view text, - beast::Journal::MessagePoolNode owner, + beast::Journal::StringBuffer text, bool console); std::string @@ -280,12 +281,16 @@ class Logs // Wraps a Journal::Stream to skip evaluation of // expensive argument lists if the stream is not active. #ifndef JLOG +#define JLOG_JOIN_(a,b) a##b +#define JLOG_JOIN(a,b) JLOG_JOIN_(a,b) +#define JLOG_UNIQUE(base) JLOG_JOIN(base, __LINE__) // line-based unique name + #define JLOG(x) \ - if (!x) \ - { \ - } \ - else \ - x + if (auto JLOG_UNIQUE(stream) = (x); !JLOG_UNIQUE(stream)) \ + { \ + } \ + else \ + JLOG_UNIQUE(stream) #endif #ifndef CLOG diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 1c30644de48..bcbca73ad64 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -35,164 +35,6 @@ #include #include -namespace beast { - -class StringBufferPool { -public: - // ----- Empty index marker ----- - static constexpr std::uint32_t kEmptyIdx = std::numeric_limits::max(); - - // ----- Single-word CAS target: {tag | idx} with pack/unpack ----- - struct Head { - std::uint32_t tag; - std::uint32_t idx; // kEmptyIdx means empty - - static std::uint64_t pack(Head h) noexcept { - return (std::uint64_t(h.tag) << 32) | h.idx; - } - static Head unpack(std::uint64_t v) noexcept { - return Head{ std::uint32_t(v >> 32), std::uint32_t(v) }; - } - }; - - // ----- Internal node ----- - struct Node { - std::uint32_t next_idx{kEmptyIdx}; - std::uint32_t self_idx{kEmptyIdx}; - std::string buf{}; - }; - static_assert(std::is_standard_layout_v, "Node must be standard layout"); - - // ----- User-facing move-only RAII handle ----- - class Handle { - public: - Handle() = default; - Handle(Handle&& other) noexcept - : owner_(other.owner_), node_(other.node_) { - other.owner_ = nullptr; other.node_ = nullptr; - } - Handle& operator=(Handle&& other) noexcept { - if (this != &other) { - // Return current if still held - if (owner_ && node_) owner_->give_back(std::move(*this)); - owner_ = other.owner_; - node_ = other.node_; - other.owner_ = nullptr; - other.node_ = nullptr; - } - return *this; - } - - Handle(const Handle&) = delete; - Handle& operator=(const Handle&) = delete; - - ~Handle() noexcept { - if (owner_ && node_) owner_->give_back(std::move(*this)); - } - - bool valid() const noexcept { return node_ != nullptr; } - std::string& string() noexcept { return node_->buf; } - const std::string& string() const noexcept { return node_->buf; } - - private: - friend class StringBufferPool; - Handle(StringBufferPool* owner, Node* n) : owner_(owner), node_(n) {} - - StringBufferPool* owner_ = nullptr; - Node* node_ = nullptr; - }; - - explicit StringBufferPool(std::uint32_t grow_by = 20) - : grow_by_(grow_by), head_(Head::pack({0, kEmptyIdx})) {} - - // Rent a buffer; grows on demand. Returns move-only RAII handle. - Handle rent() { - for (;;) { - std::uint64_t old64 = head_.load(std::memory_order_acquire); - Head old = Head::unpack(old64); - if (old.idx == kEmptyIdx) { grow_(); continue; } // rare slow path - - Node& n = nodes_[old.idx]; - std::uint32_t next = n.next_idx; - - Head neu{ std::uint32_t(old.tag + 1), next }; - if (head_.compare_exchange_weak(old64, Head::pack(neu), - std::memory_order_acq_rel, - std::memory_order_acquire)) { - return {this, &n}; - } - } - } - -private: - // Only the pool/handle can call this - void give_back(Handle&& h) noexcept { - Node* node = h.node_; - if (!node) return; // already invalid - const std::uint32_t idx = node->self_idx; - - node->buf.clear(); - - for (;;) { - std::uint64_t old64 = head_.load(std::memory_order_acquire); - Head old = Head::unpack(old64); - - node->next_idx = old.idx; - - Head neu{ std::uint32_t(old.tag + 1), idx }; - if (head_.compare_exchange_weak(old64, Head::pack(neu), - std::memory_order_acq_rel, - std::memory_order_acquire)) { - // Invalidate handle (prevents double return) - h.owner_ = nullptr; - h.node_ = nullptr; - return; - } - } - } - - void grow_() { - if (Head::unpack(head_.load(std::memory_order_acquire)).idx != kEmptyIdx) return; - std::scoped_lock lk(grow_mu_); - if (Head::unpack(head_.load(std::memory_order_acquire)).idx != kEmptyIdx) return; - - const std::uint32_t base = static_cast(nodes_.size()); - nodes_.resize(base + grow_by_); // indices [base .. base+grow_by_-1] - - // Init nodes and local chain - for (std::uint32_t i = 0; i < grow_by_; ++i) { - std::uint32_t idx = base + i; - Node& n = nodes_[idx]; - n.self_idx = idx; - n.next_idx = (i + 1 < grow_by_) ? (idx + 1) : kEmptyIdx; - } - - // Splice chain onto global head: [base .. base+grow_by_-1] - const std::uint32_t chain_head = base; - const std::uint32_t chain_tail = base + grow_by_ - 1; - - for (;;) { - std::uint64_t old64 = head_.load(std::memory_order_acquire); - Head old = Head::unpack(old64); - - nodes_[chain_tail].next_idx = old.idx; // tail -> old head - Head neu{ std::uint32_t(old.tag + 1), chain_head }; - - if (head_.compare_exchange_weak(old64, Head::pack(neu), - std::memory_order_acq_rel, - std::memory_order_acquire)) { - break; - } - } - } - - const std::uint32_t grow_by_; - std::atomic head_; // single 64-bit CAS (Head packed) - std::mutex grow_mu_; // only during growth - std::deque nodes_; // stable storage for nodes/strings -}; -} // namespace beast - namespace ripple::log { template class LogParameter @@ -248,102 +90,107 @@ namespace detail { class SimpleJsonWriter { public: - explicit SimpleJsonWriter(std::string& buffer) : buffer_(buffer) + explicit SimpleJsonWriter(std::string* buffer) : buffer_(buffer) { } + SimpleJsonWriter() = default; + + SimpleJsonWriter(SimpleJsonWriter const& other) = default; + SimpleJsonWriter& operator=(SimpleJsonWriter const& other) = default; + std::string& - buffer() { return buffer_; } + buffer() { return *buffer_; } void startObject() const { - buffer_.push_back('{'); + buffer_->push_back('{'); } void endObject() const { using namespace std::string_view_literals; - if (buffer_.back() == ',') - buffer_.pop_back(); - buffer_.append("},"sv); + if (buffer_->back() == ',') + buffer_->pop_back(); + buffer_->append("},"sv); } void writeKey(std::string_view key) const { writeString(key); - buffer_.back() = ':'; + buffer_->back() = ':'; } void startArray() const { - buffer_.push_back('['); + buffer_->push_back('['); } void endArray() const { using namespace std::string_view_literals; - if (buffer_.back() == ',') - buffer_.pop_back(); - buffer_.append("],"sv); + if (buffer_->back() == ',') + buffer_->pop_back(); + buffer_->append("],"sv); } void writeString(std::string_view str) const { using namespace std::string_view_literals; - buffer_.push_back('"'); - escape(str, buffer_); - buffer_.append("\","sv); + buffer_->push_back('"'); + escape(str, *buffer_); + buffer_->append("\","sv); } std::string_view writeInt(std::int32_t val) const { - return pushNumber(val, buffer_); + return pushNumber(val, *buffer_); } std::string_view writeInt(std::int64_t val) const { - return pushNumber(val, buffer_); + return pushNumber(val, *buffer_); } std::string_view writeUInt(std::uint32_t val) const { - return pushNumber(val, buffer_); + return pushNumber(val, *buffer_); } std::string_view writeUInt(std::uint64_t val) const { - return pushNumber(val, buffer_); + return pushNumber(val, *buffer_); } std::string_view writeDouble(double val) const { - return pushNumber(val, buffer_); + return pushNumber(val, *buffer_); } std::string_view writeBool(bool val) const { using namespace std::string_view_literals; auto str = val ? "true,"sv : "false,"sv; - buffer_.append(str); + buffer_->append(str); return str; } void writeNull() const { using namespace std::string_view_literals; - buffer_.append("null,"sv); + buffer_->append("null,"sv); } void writeRaw(std::string_view str) const { - buffer_.append(str); + buffer_->append(str); } void finish() { - buffer_.pop_back(); + buffer_->pop_back(); } private: @@ -427,7 +274,7 @@ class SimpleJsonWriter buffer.append(chunk, p - chunk); } - std::string& buffer_; + std::string* buffer_ = nullptr; }; } // namespace detail @@ -482,23 +329,141 @@ class Journal class Sink; - using MessagePoolNode = lockfree::queue::Node*; + class StringBufferPool { + public: + static constexpr std::uint32_t kEmptyIdx = std::numeric_limits::max(); + + struct Head { + std::uint32_t tag; + std::uint32_t idx; // kEmptyIdx means empty + }; + + struct Node { + std::uint32_t next_idx{kEmptyIdx}; + std::uint32_t self_idx{kEmptyIdx}; + std::string buf{}; + }; + + class StringBuffer + { + public: + StringBuffer() = default; + + std::string& + str() { return node_->buf; } + + private: + StringBuffer(StringBufferPool* owner, Node* node) + : owner_(owner), node_(node) {} + + StringBufferPool* owner_ = nullptr; + Node* node_ = nullptr; + + friend class StringBufferPool; + }; + + explicit StringBufferPool(std::uint32_t grow_by = 20) + : growBy_(grow_by), head_({0, kEmptyIdx}) {} + + // Rent a buffer; grows on demand. Returns move-only RAII handle. + StringBuffer rent() { + for (;;) { + auto old = head_.load(std::memory_order_acquire); + if (old.idx == kEmptyIdx) { grow(); continue; } // rare slow path + + Node& n = nodes_[old.idx]; + std::uint32_t next = n.next_idx; + + Head neu{ old.tag + 1, next }; + if (head_.compare_exchange_weak(old, neu, + std::memory_order_acq_rel, + std::memory_order_acquire)) { + return {this, &n}; + } + } + } + + // Only the pool/handle can call this + void giveBack(StringBuffer&& h) noexcept { + Node* node = h.node_; + if (!node) return; // already invalid + const std::uint32_t idx = node->self_idx; + + for (;;) { + auto old = head_.load(std::memory_order_acquire); + + node->next_idx = old.idx; + + Head neu{ std::uint32_t(old.tag + 1), idx }; + if (head_.compare_exchange_weak(old, neu, + std::memory_order_acq_rel, + std::memory_order_acquire)) { + // Invalidate handle (prevents double return) + h.owner_ = nullptr; + h.node_ = nullptr; + return; + } + } + } + + private: + + void grow() { + if (head_.load(std::memory_order_acquire).idx != kEmptyIdx) return; + std::scoped_lock lk(growMutex_); + if (head_.load(std::memory_order_acquire).idx != kEmptyIdx) return; + + auto base = static_cast(nodes_.size()); + nodes_.resize(base + growBy_); + + // Init nodes and local chain + for (std::uint32_t i = 0; i < growBy_; ++i) { + std::uint32_t idx = base + i; + Node& n = nodes_[idx]; + n.self_idx = idx; + n.next_idx = (i + 1 < growBy_) ? (idx + 1) : kEmptyIdx; + } + + // Splice chain onto global head: [base .. base+grow_by_-1] + const std::uint32_t chain_head = base; + const std::uint32_t chain_tail = base + growBy_ - 1; + + for (;;) { + auto old = head_.load(std::memory_order_acquire); + + nodes_[chain_tail].next_idx = old.idx; // tail -> old head + Head neu{ std::uint32_t(old.tag + 1), chain_head }; + + if (head_.compare_exchange_weak(old, neu, + std::memory_order_acq_rel, + std::memory_order_acquire)) { + break; + } + } + } + + const std::uint32_t growBy_; + + // single 64-bit CAS + std::atomic head_; + + // only during growth + std::mutex growMutex_; + + // stable storage for nodes/strings + std::deque nodes_; + }; + using StringBuffer = StringBufferPool::StringBuffer; class JsonLogContext { - MessagePoolNode messageBuffer_; - detail::SimpleJsonWriter messageParamsWriter_; + StringBuffer messageBuffer_; + detail::SimpleJsonWriter jsonWriter_; bool hasMessageParams_ = false; - + bool messageBufferHandedOut_ = true; public: - explicit JsonLogContext() - : messageBuffer_(rentFromPool()) - , messageParamsWriter_(messageBuffer_->data) - { - messageBuffer_->data.reserve(1024 * 5); - } - MessagePoolNode + StringBuffer messageBuffer() { return messageBuffer_; } void @@ -524,11 +489,14 @@ class Journal detail::SimpleJsonWriter& writer() { - return messageParamsWriter_; + return jsonWriter_; } void - reset( + finish(); + + void + start( std::source_location location, severities::Severity severity, std::string_view moduleName, @@ -545,7 +513,7 @@ class Journal static std::shared_mutex globalLogAttributesMutex_; static bool jsonLogsEnabled_; - static lockfree::queue messagePool_; + static StringBufferPool messagePool_; static thread_local JsonLogContext currentJsonLogContext_; // Invariant: m_sink always points to a valid Sink @@ -556,25 +524,20 @@ class Journal std::source_location location, severities::Severity severity) const; - static MessagePoolNode + static StringBuffer formatLog(std::string const& message); public: //-------------------------------------------------------------------------- - static MessagePoolNode + static StringBuffer rentFromPool() { - auto node = messagePool_.pop(); - if (!node) - { - node = new lockfree::queue::Node(); - } - return node; + return messagePool_.rent(); } static void - returnMessageNode(MessagePoolNode node) { messagePool_.push(node); } + returnStringBuffer(StringBuffer&& node) { messagePool_.giveBack(std::move(node)); } static void enableStructuredJournal(); @@ -625,7 +588,7 @@ class Journal level is below the current threshold(). */ virtual void - write(Severity level, std::string_view text, MessagePoolNode owner = nullptr) = 0; + write(Severity level, StringBuffer text) = 0; /** Bypass filter and write text to the sink at the specified severity. * Always write the message, but maintain the same formatting as if @@ -635,7 +598,7 @@ class Journal * @param text Text to write to sink. */ virtual void - writeAlways(Severity level, std::string_view text, MessagePoolNode owner = nullptr) = 0; + writeAlways(Severity level, StringBuffer text) = 0; private: Severity thresh_; @@ -813,7 +776,7 @@ class Journal : name_(other.name_), m_sink(other.m_sink) { std::string buffer{other.attributes_}; - detail::SimpleJsonWriter writer{buffer}; + detail::SimpleJsonWriter writer{&buffer}; if (other.attributes_.empty() && jsonLogsEnabled_) { writer.startObject(); @@ -838,7 +801,7 @@ class Journal { std::string buffer; buffer.reserve(128); - detail::SimpleJsonWriter writer{buffer}; + detail::SimpleJsonWriter writer{&buffer}; if (jsonLogsEnabled_) { writer.startObject(); @@ -954,7 +917,7 @@ class Journal std::unique_lock lock(globalLogAttributesMutex_); globalLogAttributes_.reserve(1024); auto isEmpty = globalLogAttributes_.empty(); - detail::SimpleJsonWriter writer{globalLogAttributes_}; + detail::SimpleJsonWriter writer{&globalLogAttributes_}; if (isEmpty && jsonLogsEnabled_) { writer.startObject(); diff --git a/include/xrpl/beast/utility/WrappedSink.h b/include/xrpl/beast/utility/WrappedSink.h index 6024e6b7972..8e083c467ec 100644 --- a/include/xrpl/beast/utility/WrappedSink.h +++ b/include/xrpl/beast/utility/WrappedSink.h @@ -88,17 +88,19 @@ class WrappedSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override + write(beast::severities::Severity level, Journal::StringBuffer text) override { using beast::Journal; - sink_.write(level, prefix_ + std::string(text), owner); + text.str() = prefix_ + text.str(); + sink_.write(level, text); } void - writeAlways(severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override + writeAlways(severities::Severity level, Journal::StringBuffer text) override { using beast::Journal; - sink_.writeAlways(level, prefix_ + std::string(text), owner); + text.str() = prefix_ + text.str(); + sink_.writeAlways(level, text); } }; diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index 07fdffe6827..7461086d6c8 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -53,18 +53,18 @@ Logs::Sink::Sink( } void -Logs::Sink::write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner) +Logs::Sink::write(beast::severities::Severity level, beast::Journal::StringBuffer text) { if (level < threshold()) return; - logs_.write(level, partition_, text, owner, console()); + logs_.write(level, partition_, text, console()); } void -Logs::Sink::writeAlways(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner) +Logs::Sink::writeAlways(beast::severities::Severity level, beast::Journal::StringBuffer text) { - logs_.write(level, partition_, text, owner, console()); + logs_.write(level, partition_, text, console()); } //------------------------------------------------------------------------------ @@ -119,7 +119,7 @@ Logs::File::close() } void -Logs::File::write(std::string&& text) +Logs::File::write(std::string const& text) { if (m_stream.has_value()) m_stream->write(text.data(), text.size()); @@ -141,10 +141,8 @@ Logs::~Logs() { // Signal log thread to stop and wait for it to finish { - std::lock_guard lock(logMutex_); stopLogThread_ = true; } - logCondition_.notify_all(); if (logThread_.joinable()) logThread_.join(); @@ -202,27 +200,25 @@ void Logs::write( beast::severities::Severity level, std::string const& partition, - std::string_view text, - beast::Journal::MessagePoolNode owner, + beast::Journal::StringBuffer text, bool console) { std::string s; - std::string_view result = text; + std::string_view result = text.str(); if (!beast::Journal::isStructuredJournalEnabled()) { - format(s, text, level, partition); - result = s; + format(s, text.str(), level, partition); + text.str() = s; + result = text.str(); } // if (!silent_) - // std::cerr << s << '\n'; + // std::cerr << result << '\n'; - // Get a node from the pool or create a new one - if (!owner) return; - messages_.push(owner); + messages_.push(text); // Signal log thread that new messages are available - logCondition_.notify_one(); + // logCondition_.notify_one(); // Add to batch buffer for file output if (0) { @@ -291,42 +287,25 @@ Logs::flushBatchUnsafe() void Logs::logThreadWorker() { - beast::lockfree::queue::Node* node; - while (!stopLogThread_) { - std::unique_lock lock(logMutex_); - - // Wait for messages or stop signal - logCondition_.wait(lock, [this] { - return stopLogThread_ || !messages_.empty(); - }); - + std::this_thread::sleep_for(FLUSH_INTERVAL); + + beast::Journal::StringBuffer buffer; // Process all available messages - while ((node = messages_.pop())) + while (messages_.pop(buffer)) { - // Write to file - file_.write(std::move(node->data)); - // Also write to console if not silent if (!silent_) - std::cerr << node->data << '\n'; - + std::cerr << buffer.str() << '\n'; + + // Write to file + file_.write(buffer.str()); + // Return node to pool for reuse - beast::Journal::returnMessageNode(node); + beast::Journal::returnStringBuffer(std::move(buffer)); } } - - // Process any remaining messages on shutdown - while ((node = messages_.pop())) - { - file_.write(std::move(node->data)); - if (!silent_) - std::cerr << node->data << '\n'; - - // Return node to pool for reuse - beast::Journal::returnMessageNode(node); - } } std::string diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index b7a7c7ef187..6a932ac8de8 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -113,7 +113,7 @@ fastTimestampToString(std::int64_t milliseconds_since_epoch) std::string Journal::globalLogAttributes_; std::shared_mutex Journal::globalLogAttributesMutex_; bool Journal::jsonLogsEnabled_ = false; -lockfree::queue Journal::messagePool_{}; +Journal::StringBufferPool Journal::messagePool_{}; thread_local Journal::JsonLogContext Journal::currentJsonLogContext_{}; //------------------------------------------------------------------------------ @@ -157,12 +157,12 @@ class NullJournalSink : public Journal::Sink } void - write(severities::Severity, std::string_view, Journal::MessagePoolNode = nullptr) override + write(severities::Severity, Journal::StringBuffer) override { } void - writeAlways(severities::Severity, std::string_view, Journal::MessagePoolNode = nullptr) override + writeAlways(severities::Severity, Journal::StringBuffer) override { } }; @@ -205,7 +205,7 @@ severities::to_string(Severity severity) } void -Journal::JsonLogContext::reset( +Journal::JsonLogContext::start( std::source_location location, severities::Severity severity, std::string_view moduleName, @@ -223,7 +223,16 @@ Journal::JsonLogContext::reset( }; thread_local ThreadIdStringInitializer const threadId; - messageBuffer_->data.clear(); + if (!messageBufferHandedOut_) + { + returnStringBuffer(std::move(messageBuffer_)); + messageBufferHandedOut_ = true; + } + messageBuffer_ = rentFromPool(); + messageBufferHandedOut_ = false; + messageBuffer_.str().reserve(1024 * 5); + messageBuffer_.str().clear(); + jsonWriter_ = detail::SimpleJsonWriter{&messageBuffer_.str()}; if (!jsonLogsEnabled_) { @@ -284,15 +293,23 @@ Journal::JsonLogContext::reset( hasMessageParams_ = false; } +void +Journal::JsonLogContext::finish() +{ + messageBufferHandedOut_ = true; + messageBuffer_ = {}; + jsonWriter_ = {}; +} + void Journal::initMessageContext( std::source_location location, severities::Severity severity) const { - currentJsonLogContext_.reset(location, severity, name_, attributes_); + currentJsonLogContext_.start(location, severity, name_, attributes_); } -Journal::MessagePoolNode +Journal::StringBuffer Journal::formatLog(std::string const& message) { if (!jsonLogsEnabled_) @@ -397,7 +414,8 @@ Journal::ScopedStream::~ScopedStream() s = ""; auto messageHandle = formatLog(s); - m_sink.write(m_level, messageHandle->data, messageHandle); + m_sink.write(m_level, messageHandle); + currentJsonLogContext_.finish(); } } @@ -407,12 +425,4 @@ Journal::ScopedStream::operator<<(std::ostream& manip(std::ostream&)) const return m_ostream << manip; } -//------------------------------------------------------------------------------ - -Journal::ScopedStream -Journal::Stream::operator<<(std::ostream& manip(std::ostream&)) const -{ - return {*this, manip}; -} - } // namespace beast diff --git a/src/test/beast/beast_Journal_test.cpp b/src/test/beast/beast_Journal_test.cpp index e50d47b2e86..43f5d1668e6 100644 --- a/src/test/beast/beast_Journal_test.cpp +++ b/src/test/beast/beast_Journal_test.cpp @@ -48,14 +48,14 @@ class Journal_test : public unit_test::suite } void - write(severities::Severity level, std::string_view, beast::Journal::MessagePoolNode = nullptr) override + write(severities::Severity level, Journal::StringBuffer) override { if (level >= threshold()) ++m_count; } void - writeAlways(severities::Severity level, std::string_view, beast::Journal::MessagePoolNode = nullptr) override + writeAlways(severities::Severity level, Journal::StringBuffer) override { ++m_count; } diff --git a/src/test/csf/Sim.h b/src/test/csf/Sim.h index 7a606b40bf9..e25d1a17f1c 100644 --- a/src/test/csf/Sim.h +++ b/src/test/csf/Sim.h @@ -49,19 +49,19 @@ class BasicSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override + write(beast::severities::Severity level, beast::Journal::StringBuffer text) override { if (level < threshold()) return; - std::cout << clock_.now().time_since_epoch().count() << " " << text + std::cout << clock_.now().time_since_epoch().count() << " " << text.str() << std::endl; } void - writeAlways(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override + writeAlways(beast::severities::Severity level, beast::Journal::StringBuffer text) override { - std::cout << clock_.now().time_since_epoch().count() << " " << text + std::cout << clock_.now().time_since_epoch().count() << " " << text.str() << std::endl; } }; diff --git a/src/test/jtx/CaptureLogs.h b/src/test/jtx/CaptureLogs.h index 2abcc6bd251..4d3bca044b6 100644 --- a/src/test/jtx/CaptureLogs.h +++ b/src/test/jtx/CaptureLogs.h @@ -57,18 +57,18 @@ class CaptureLogs : public Logs } void - write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override + write(beast::severities::Severity level, beast::Journal::StringBuffer text) override { std::lock_guard lock(strmMutex_); - strm_ << text; + strm_ << text.str(); } void - writeAlways(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) + writeAlways(beast::severities::Severity level, beast::Journal::StringBuffer text) override { std::lock_guard lock(strmMutex_); - strm_ << text; + strm_ << text.str(); } }; diff --git a/src/test/jtx/CheckMessageLogs.h b/src/test/jtx/CheckMessageLogs.h index 18ae98b196e..63c37a66197 100644 --- a/src/test/jtx/CheckMessageLogs.h +++ b/src/test/jtx/CheckMessageLogs.h @@ -45,17 +45,17 @@ class CheckMessageLogs : public Logs } void - write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override + write(beast::severities::Severity level, beast::Journal::StringBuffer text) override { - if (text.find(owner_.msg_) != std::string::npos) + if (text.str().find(owner_.msg_) != std::string::npos) *owner_.pFound_ = true; } void - writeAlways(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) + writeAlways(beast::severities::Severity level, beast::Journal::StringBuffer text) override { - write(level, text, owner); + write(level, text); } }; diff --git a/src/test/server/Server_test.cpp b/src/test/server/Server_test.cpp index 3ebed8b699f..4aa76b715ed 100644 --- a/src/test/server/Server_test.cpp +++ b/src/test/server/Server_test.cpp @@ -92,19 +92,19 @@ class Server_test : public beast::unit_test::suite } void - write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override + write(beast::severities::Severity level, beast::Journal::StringBuffer text) override { if (level < threshold()) return; - suite_.log << text << std::endl; + suite_.log << text.str() << std::endl; } void - writeAlways(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) + writeAlways(beast::severities::Severity level, beast::Journal::StringBuffer text) override { - suite_.log << text << std::endl; + suite_.log << text.str() << std::endl; } }; diff --git a/src/test/unit_test/SuiteJournal.h b/src/test/unit_test/SuiteJournal.h index fc0f85d69a4..f4858cd7809 100644 --- a/src/test/unit_test/SuiteJournal.h +++ b/src/test/unit_test/SuiteJournal.h @@ -49,25 +49,24 @@ class SuiteJournalSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override; + write(beast::severities::Severity level, beast::Journal::StringBuffer text) override; void - writeAlways(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override; + writeAlways(beast::severities::Severity level, beast::Journal::StringBuffer text) override; }; inline void -SuiteJournalSink::write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner) +SuiteJournalSink::write(beast::severities::Severity level, beast::Journal::StringBuffer text) { // Only write the string if the level at least equals the threshold. if (level >= threshold()) - writeAlways(level, text, owner); + writeAlways(level, text); } inline void SuiteJournalSink::writeAlways( beast::severities::Severity level, - std::string_view text, - beast::Journal::MessagePoolNode owner) + beast::Journal::StringBuffer text) { using namespace beast::severities; @@ -94,7 +93,7 @@ SuiteJournalSink::writeAlways( static std::mutex log_mutex; std::lock_guard lock(log_mutex); - suite_.log << s << partition_ << text << std::endl; + suite_.log << s << partition_ << text.str() << std::endl; } class SuiteJournal @@ -135,17 +134,17 @@ class StreamSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override + write(beast::severities::Severity level, beast::Journal::StringBuffer text) override { if (level < threshold()) return; - writeAlways(level, text, owner); + writeAlways(level, text); } inline void - writeAlways(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode = nullptr) override + writeAlways(beast::severities::Severity level, beast::Journal::StringBuffer text) override { - strm_ << text << std::endl; + strm_ << text.str() << std::endl; } std::stringstream const& diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index 10c2bb4c891..f48d3475301 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -53,13 +53,13 @@ class MockLogs : public Logs operator=(Sink const&) = delete; void - write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override + write(beast::severities::Severity level, beast::Journal::StringBuffer text) override { logs_.write(level, partition_, text, false); } void - writeAlways(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) + writeAlways(beast::severities::Severity level, beast::Journal::StringBuffer text) override { logs_.write(level, partition_, text, false); @@ -86,17 +86,19 @@ class MockLogs : public Logs write( beast::severities::Severity level, std::string const& partition, - std::string_view text, + beast::Journal::StringBuffer text, bool console) { std::string s; - std::string_view result = text; + std::string_view result = text.str(); if (!beast::Journal::isStructuredJournalEnabled()) { - format(s, text, level, partition); - result = s; + format(s, text.str(), level, partition); + text.str() = s; + result = text.str(); } logStream_.append(result); + beast::Journal::returnStringBuffer(std::move(text)); } }; @@ -228,7 +230,8 @@ TEST_CASE("Test JsonWriter") beast::detail::SimpleJsonWriter writer{buffer}; writer.writeString("\n\r\t123\b\f123"); - CHECK(writer.finish() == "\"\\n\\r\\t123\\b\\f123\""); + writer.finish(); + CHECK(writer.buffer() == "\"\\n\\r\\t123\\b\\f123\""); } { @@ -236,7 +239,8 @@ TEST_CASE("Test JsonWriter") beast::detail::SimpleJsonWriter writer{buffer}; writer.writeString("\t"); - CHECK(writer.finish() == "\"\\t\""); + writer.finish(); + CHECK(writer.buffer() == "\"\\t\""); } { @@ -244,7 +248,8 @@ TEST_CASE("Test JsonWriter") beast::detail::SimpleJsonWriter writer{buffer}; writer.writeString(std::string_view{"\0", 1}); - CHECK(writer.finish() == "\"\\u0000\""); + writer.finish(); + CHECK(writer.buffer() == "\"\\u0000\""); } { @@ -252,7 +257,8 @@ TEST_CASE("Test JsonWriter") beast::detail::SimpleJsonWriter writer{buffer}; writer.writeString("\"\\"); - CHECK(writer.finish() == "\"\\\"\\\\\""); + writer.finish(); + CHECK(writer.buffer() == "\"\\\"\\\\\""); } { @@ -264,7 +270,8 @@ TEST_CASE("Test JsonWriter") writer.writeBool(false); writer.writeNull(); writer.endArray(); - CHECK(writer.finish() == "[true,false,null]"); + writer.finish(); + CHECK(writer.buffer() == "[true,false,null]"); } } @@ -321,9 +328,10 @@ TEST_CASE("Test setJsonValue") log::detail::setJsonValue( writer, "testStream", {}, &stringBuf); writer.endObject(); + writer.finish(); CHECK( - writer.finish() == + writer.buffer() == R"AAA({"testBool":true,"testInt32":-1,"testUInt32":1,"testInt64":-1,"testUInt64":1,"testDouble":1.1,"testCharStar":"Char*","testStdString":"StdString","testStdStringView":"StdStringView","testToChars":"0","testStream":"0"})AAA"); } @@ -358,15 +366,15 @@ class MockSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override + write(beast::severities::Severity level, beast::Journal::StringBuffer text) override { - strm_ << text; + strm_ << text.str(); } void - writeAlways(beast::severities::Severity level, std::string_view text, beast::Journal::MessagePoolNode owner = nullptr) override + writeAlways(beast::severities::Severity level, beast::Journal::StringBuffer text) override { - strm_ << text; + strm_ << text.str(); } }; diff --git a/src/xrpld/app/consensus/RCLConsensus.cpp b/src/xrpld/app/consensus/RCLConsensus.cpp index 8dfd48c761b..c43ed1fdc5e 100644 --- a/src/xrpld/app/consensus/RCLConsensus.cpp +++ b/src/xrpld/app/consensus/RCLConsensus.cpp @@ -1132,10 +1132,10 @@ RclConsensusLogger::~RclConsensusLogger() << std::setw(3) << std::setfill('0') << (duration.count() % 1000) << "s. " << ss_->str(); + auto node = beast::Journal::rentFromPool(); if (beast::Journal::isStructuredJournalEnabled()) { - auto node = beast::Journal::rentFromPool(); - beast::detail::SimpleJsonWriter writer{node->data}; + beast::detail::SimpleJsonWriter writer{&node.str()}; writer.startObject(); writer.writeKey("Msg"); writer.writeString(outSs.str()); @@ -1143,12 +1143,12 @@ RclConsensusLogger::~RclConsensusLogger() writer.writeString(to_string(std::chrono::system_clock::now())); writer.endObject(); writer.finish(); - j_.sink().writeAlways(beast::severities::kInfo, writer.buffer(), node); } else { - j_.sink().writeAlways(beast::severities::kInfo, outSs.str()); + node.str() = outSs.str(); } + j_.sink().writeAlways(beast::severities::kInfo, node); } } // namespace ripple From 6376f10df7ce7b33f8c1128812daea394dac5829 Mon Sep 17 00:00:00 2001 From: JCW Date: Fri, 12 Sep 2025 18:53:56 +0100 Subject: [PATCH 79/88] Fix unit tests Signed-off-by: JCW --- src/tests/libxrpl/basics/log.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index f48d3475301..5054d309b6c 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -227,7 +227,7 @@ TEST_CASE("Test JsonWriter") { { std::string buffer; - beast::detail::SimpleJsonWriter writer{buffer}; + beast::detail::SimpleJsonWriter writer{&buffer}; writer.writeString("\n\r\t123\b\f123"); writer.finish(); @@ -236,7 +236,7 @@ TEST_CASE("Test JsonWriter") { std::string buffer; - beast::detail::SimpleJsonWriter writer{buffer}; + beast::detail::SimpleJsonWriter writer{&buffer}; writer.writeString("\t"); writer.finish(); @@ -245,7 +245,7 @@ TEST_CASE("Test JsonWriter") { std::string buffer; - beast::detail::SimpleJsonWriter writer{buffer}; + beast::detail::SimpleJsonWriter writer{&buffer}; writer.writeString(std::string_view{"\0", 1}); writer.finish(); @@ -254,7 +254,7 @@ TEST_CASE("Test JsonWriter") { std::string buffer; - beast::detail::SimpleJsonWriter writer{buffer}; + beast::detail::SimpleJsonWriter writer{&buffer}; writer.writeString("\"\\"); writer.finish(); @@ -263,7 +263,7 @@ TEST_CASE("Test JsonWriter") { std::string buffer; - beast::detail::SimpleJsonWriter writer{buffer}; + beast::detail::SimpleJsonWriter writer{&buffer}; writer.startArray(); writer.writeBool(true); @@ -304,7 +304,7 @@ TEST_CASE("Test setJsonValue") { std::ostringstream stringBuf; std::string buffer; - beast::detail::SimpleJsonWriter writer{buffer}; + beast::detail::SimpleJsonWriter writer{&buffer}; writer.startObject(); log::detail::setJsonValue(writer, "testBool", true, &stringBuf); From 129166cda551ab70efb5a281439547477457048e Mon Sep 17 00:00:00 2001 From: JCW Date: Fri, 12 Sep 2025 19:15:18 +0100 Subject: [PATCH 80/88] Fix build error Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index bcbca73ad64..8396b9497c1 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -23,6 +23,7 @@ #include #include +#include #include #include #include From d1fe8ed31d880803da4c18adeb09e483d99ccbdf Mon Sep 17 00:00:00 2001 From: JCW Date: Fri, 12 Sep 2025 23:27:57 +0100 Subject: [PATCH 81/88] Fix error Signed-off-by: JCW --- include/xrpl/beast/utility/Journal.h | 42 ++++++++++++++++----- src/libxrpl/beast/utility/beast_Journal.cpp | 34 +++++++++++------ 2 files changed, 54 insertions(+), 22 deletions(-) diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 8396b9497c1..40098ebd0e3 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -461,9 +461,14 @@ class Journal StringBuffer messageBuffer_; detail::SimpleJsonWriter jsonWriter_; bool hasMessageParams_ = false; - bool messageBufferHandedOut_ = true; + std::size_t messageOffset_ = 0; public: + JsonLogContext() + : messageBuffer_(rentFromPool()) + , jsonWriter_(&messageBuffer_.str()) + {} + StringBuffer messageBuffer() { return messageBuffer_; } @@ -493,6 +498,9 @@ class Journal return jsonWriter_; } + void + reuseJson(); + void finish(); @@ -739,11 +747,32 @@ class Journal /** Output stream support. */ /** @{ */ ScopedStream - operator<<(std::ostream& manip(std::ostream&)) const; + operator<<(std::ostream& manip(std::ostream&)) const && + { + return {*this, manip}; + } template ScopedStream - operator<<(T const& t) const; + operator<<(T const& t) const && + { + return {*this, t}; + } + + ScopedStream + operator<<(std::ostream& manip(std::ostream&)) const & + { + currentJsonLogContext_.reuseJson(); + return {*this, manip}; + } + + template + ScopedStream + operator<<(T const& t) const & + { + currentJsonLogContext_.reuseJson(); + return {*this, t}; + } /** @} */ private: @@ -955,13 +984,6 @@ Journal::ScopedStream::operator<<(T const& t) const //------------------------------------------------------------------------------ -template -Journal::ScopedStream -Journal::Stream::operator<<(T const& t) const -{ - return {*this, t}; -} - namespace detail { template > diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index 6a932ac8de8..f37aa05bb5e 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -223,19 +223,13 @@ Journal::JsonLogContext::start( }; thread_local ThreadIdStringInitializer const threadId; - if (!messageBufferHandedOut_) - { - returnStringBuffer(std::move(messageBuffer_)); - messageBufferHandedOut_ = true; - } - messageBuffer_ = rentFromPool(); - messageBufferHandedOut_ = false; - messageBuffer_.str().reserve(1024 * 5); + messageOffset_ = 0; messageBuffer_.str().clear(); jsonWriter_ = detail::SimpleJsonWriter{&messageBuffer_.str()}; if (!jsonLogsEnabled_) { + messageBuffer_.str() = journalAttributes; return; } @@ -293,12 +287,28 @@ Journal::JsonLogContext::start( hasMessageParams_ = false; } +void +Journal::JsonLogContext::reuseJson() +{ + messageOffset_ = messageBuffer_.str().size(); +} void Journal::JsonLogContext::finish() { - messageBufferHandedOut_ = true; - messageBuffer_ = {}; - jsonWriter_ = {}; + if (messageOffset_ != 0) + { + auto buffer = rentFromPool(); + std::string_view json{messageBuffer_.str()}; + buffer.str() = json.substr(0, messageOffset_); + messageBuffer_ = buffer; + } + else + { + messageBuffer_ = rentFromPool(); + } + + messageBuffer_.str().reserve(1024 * 5); + jsonWriter_ = detail::SimpleJsonWriter{&messageBuffer_.str()}; } void @@ -414,8 +424,8 @@ Journal::ScopedStream::~ScopedStream() s = ""; auto messageHandle = formatLog(s); - m_sink.write(m_level, messageHandle); currentJsonLogContext_.finish(); + m_sink.write(m_level, messageHandle); } } From 23029ab2b66de334f8e9647d4d2333561faee45f Mon Sep 17 00:00:00 2001 From: JCW Date: Fri, 12 Sep 2025 23:39:29 +0100 Subject: [PATCH 82/88] Optimise Signed-off-by: JCW --- include/xrpl/basics/Log.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/xrpl/basics/Log.h b/include/xrpl/basics/Log.h index cabdbc70d50..dc88aa51a38 100644 --- a/include/xrpl/basics/Log.h +++ b/include/xrpl/basics/Log.h @@ -290,7 +290,7 @@ class Logs { \ } \ else \ - JLOG_UNIQUE(stream) + std::move(JLOG_UNIQUE(stream)) #endif #ifndef CLOG From 56a45506eb0ccfbe144e0828b48efb8343a6feed Mon Sep 17 00:00:00 2001 From: JCW Date: Sun, 14 Sep 2025 09:11:29 +0100 Subject: [PATCH 83/88] Optimise Signed-off-by: JCW --- src/xrpld/app/paths/Flow.cpp | 10 +++++----- src/xrpld/app/rdb/backend/detail/Node.cpp | 3 +-- src/xrpld/overlay/Slot.h | 2 +- src/xrpld/overlay/detail/PeerImp.cpp | 4 ++-- src/xrpld/shamap/detail/SHAMap.cpp | 6 +++--- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/xrpld/app/paths/Flow.cpp b/src/xrpld/app/paths/Flow.cpp index 3b14b8b968d..79b3e4434bf 100644 --- a/src/xrpld/app/paths/Flow.cpp +++ b/src/xrpld/app/paths/Flow.cpp @@ -111,17 +111,17 @@ flow( ammContext.setMultiPath(strands.size() > 1); - if (j.trace()) + if (auto stream = j.trace()) { - j.trace() << "\nsrc: " << src << "\ndst: " << dst + stream << "\nsrc: " << src << "\ndst: " << dst << "\nsrcIssue: " << srcIssue << "\ndstIssue: " << dstIssue; - j.trace() << "\nNumStrands: " << strands.size(); + stream << "\nNumStrands: " << strands.size(); for (auto const& curStrand : strands) { - j.trace() << "NumSteps: " << curStrand.size(); + stream << "NumSteps: " << curStrand.size(); for (auto const& step : curStrand) { - j.trace() << '\n' << *step << '\n'; + stream << '\n' << *step << '\n'; } } } diff --git a/src/xrpld/app/rdb/backend/detail/Node.cpp b/src/xrpld/app/rdb/backend/detail/Node.cpp index 6a0544091b4..4e542988f6d 100644 --- a/src/xrpld/app/rdb/backend/detail/Node.cpp +++ b/src/xrpld/app/rdb/backend/detail/Node.cpp @@ -565,8 +565,7 @@ getHashesByIndex( if (!lhO || !phO) { - auto stream = j.trace(); - JLOG(stream) << "Don't have ledger " << ledgerIndex; + JLOG(j.trace()) << "Don't have ledger " << ledgerIndex; return {}; } diff --git a/src/xrpld/overlay/Slot.h b/src/xrpld/overlay/Slot.h index ea9fc3285ba..88afb040753 100644 --- a/src/xrpld/overlay/Slot.h +++ b/src/xrpld/overlay/Slot.h @@ -406,7 +406,7 @@ Slot::update( v.state = PeerState::Selected; else if (v.state != PeerState::Squelched) { - if (journal_.trace()) + if (journal_.active(beast::severities::kTrace)) str << k << " "; v.state = PeerState::Squelched; std::chrono::seconds duration = diff --git a/src/xrpld/overlay/detail/PeerImp.cpp b/src/xrpld/overlay/detail/PeerImp.cpp index bcb3a3cdaf2..1d4bcad1a28 100644 --- a/src/xrpld/overlay/detail/PeerImp.cpp +++ b/src/xrpld/overlay/detail/PeerImp.cpp @@ -993,7 +993,7 @@ PeerImp::onReadMessage(error_code ec, std::size_t bytes_transferred) if (auto stream = journal_.trace()) { - stream << "onReadMessage: " + std::move(stream) << "onReadMessage: " << (bytes_transferred > 0 ? to_string(bytes_transferred) + " bytes" : ""); @@ -1075,7 +1075,7 @@ PeerImp::onWriteMessage(error_code ec, std::size_t bytes_transferred) if (auto stream = journal_.trace()) { - stream << "onWriteMessage: " + std::move(stream) << "onWriteMessage: " << (bytes_transferred > 0 ? to_string(bytes_transferred) + " bytes" : ""); diff --git a/src/xrpld/shamap/detail/SHAMap.cpp b/src/xrpld/shamap/detail/SHAMap.cpp index d2415a2ff24..3f6cb7c7ea9 100644 --- a/src/xrpld/shamap/detail/SHAMap.cpp +++ b/src/xrpld/shamap/detail/SHAMap.cpp @@ -950,15 +950,15 @@ SHAMap::fetchRoot(SHAMapHash const& hash, SHAMapSyncFilter* filter) { if (type_ == SHAMapType::TRANSACTION) { - stream << "Fetch root TXN node " << hash; + std::move(stream) << "Fetch root TXN node " << hash; } else if (type_ == SHAMapType::STATE) { - stream << "Fetch root STATE node " << hash; + std::move(stream) << "Fetch root STATE node " << hash; } else { - stream << "Fetch root SHAMap node " << hash; + std::move(stream) << "Fetch root SHAMap node " << hash; } } From 6de7802001ce93fa5f33743e69a5365f0aaba673 Mon Sep 17 00:00:00 2001 From: JCW Date: Fri, 26 Sep 2025 15:21:00 +0100 Subject: [PATCH 84/88] Remove unrelated changes Signed-off-by: JCW --- include/xrpl/basics/Log.h | 64 ++----- include/xrpl/beast/utility/Journal.h | 155 +---------------- include/xrpl/beast/utility/WrappedSink.h | 10 +- src/libxrpl/basics/Log.cpp | 176 +++++--------------- src/libxrpl/beast/utility/beast_Journal.cpp | 120 ++----------- src/test/beast/beast_Journal_test.cpp | 4 +- src/test/csf/Sim.h | 9 +- src/test/jtx/CaptureLogs.h | 9 +- src/test/jtx/CheckMessageLogs.h | 7 +- src/test/server/Server_test.cpp | 9 +- src/test/unit_test/SuiteJournal.h | 20 ++- src/tests/libxrpl/CMakeLists.txt | 2 - src/xrpld/app/consensus/RCLConsensus.cpp | 13 +- src/xrpld/app/consensus/RCLConsensus.h | 4 +- src/xrpld/app/main/Application.cpp | 1 - src/xrpld/app/paths/Flow.cpp | 11 +- src/xrpld/core/detail/Config.cpp | 2 - src/xrpld/overlay/detail/PeerImp.cpp | 8 - src/xrpld/overlay/detail/PeerImp.h | 8 +- 19 files changed, 138 insertions(+), 494 deletions(-) diff --git a/include/xrpl/basics/Log.h b/include/xrpl/basics/Log.h index dc88aa51a38..2cba2ba1fb7 100644 --- a/include/xrpl/basics/Log.h +++ b/include/xrpl/basics/Log.h @@ -26,19 +26,12 @@ #include #include -#include - -#include -#include -#include -#include #include #include #include #include -#include -#include #include +#include namespace ripple { @@ -76,10 +69,11 @@ class Logs operator=(Sink const&) = delete; void - write(beast::severities::Severity level, beast::Journal::StringBuffer text) override; + write(beast::severities::Severity level, std::string const& text) + override; void - writeAlways(beast::severities::Severity level, beast::Journal::StringBuffer text) + writeAlways(beast::severities::Severity level, std::string const& text) override; }; @@ -137,16 +131,22 @@ class Logs Does nothing if there is no associated system file. */ void - write(std::string const& str); + write(std::string_view text); + + /** write to the log file and append an end of line marker. + Does nothing if there is no associated system file. + */ + void + writeln(std::string_view text); /** @} */ private: - std::optional m_stream; + std::unique_ptr m_stream; boost::filesystem::path m_path; }; - std::mutex mutable sinkSetMutex_; + std::mutex mutable mutex_; std::map< std::string, std::unique_ptr, @@ -156,24 +156,6 @@ class Logs File file_; bool silent_ = false; - // Batching members - mutable std::mutex batchMutex_; - boost::lockfree::queue> messages_; - static constexpr size_t BATCH_BUFFER_SIZE = 64 * 1024; // 64KB buffer - std::array batchBuffer_{}; - std::span writeBuffer_; // Points to available write space - std::span readBuffer_; // Points to data ready to flush - - // Log thread members - std::thread logThread_; - std::atomic stopLogThread_; - std::mutex logMutex_; - std::condition_variable logCondition_; - -private: - std::chrono::steady_clock::time_point lastFlush_ = - std::chrono::steady_clock::now(); - public: Logs(beast::severities::Severity level); @@ -181,7 +163,7 @@ class Logs Logs& operator=(Logs const&) = delete; - virtual ~Logs(); // Need to flush on destruction + virtual ~Logs() = default; bool open(boost::filesystem::path const& pathToLogFile); @@ -201,10 +183,7 @@ class Logs } beast::Journal - journal(std::string const& name) - { - return beast::Journal{get(name), name}; - } + journal(std::string const& name); beast::severities::Severity threshold() const; @@ -219,15 +198,12 @@ class Logs write( beast::severities::Severity level, std::string const& partition, - beast::Journal::StringBuffer text, + std::string const& text, bool console); std::string rotate(); - void - flushBatch(); - /** * Set flag to write logs to stderr (false) or not (true). * @@ -260,7 +236,7 @@ class Logs static void format( std::string& output, - std::string_view message, + std::string const& message, beast::severities::Severity severity, std::string const& partition); @@ -270,12 +246,6 @@ class Logs // If the message exceeds this length it will be truncated with elipses. maximumMessageCharacters = 12 * 1024 }; - - void - flushBatchUnsafe(); - - void - logThreadWorker(); }; // Wraps a Journal::Stream to skip evaluation of diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 40098ebd0e3..775e7fd0910 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -22,12 +22,10 @@ #include -#include #include #include #include #include -#include #include #include #include @@ -330,146 +328,21 @@ class Journal class Sink; - class StringBufferPool { - public: - static constexpr std::uint32_t kEmptyIdx = std::numeric_limits::max(); - - struct Head { - std::uint32_t tag; - std::uint32_t idx; // kEmptyIdx means empty - }; - - struct Node { - std::uint32_t next_idx{kEmptyIdx}; - std::uint32_t self_idx{kEmptyIdx}; - std::string buf{}; - }; - - class StringBuffer - { - public: - StringBuffer() = default; - - std::string& - str() { return node_->buf; } - - private: - StringBuffer(StringBufferPool* owner, Node* node) - : owner_(owner), node_(node) {} - - StringBufferPool* owner_ = nullptr; - Node* node_ = nullptr; - - friend class StringBufferPool; - }; - - explicit StringBufferPool(std::uint32_t grow_by = 20) - : growBy_(grow_by), head_({0, kEmptyIdx}) {} - - // Rent a buffer; grows on demand. Returns move-only RAII handle. - StringBuffer rent() { - for (;;) { - auto old = head_.load(std::memory_order_acquire); - if (old.idx == kEmptyIdx) { grow(); continue; } // rare slow path - - Node& n = nodes_[old.idx]; - std::uint32_t next = n.next_idx; - - Head neu{ old.tag + 1, next }; - if (head_.compare_exchange_weak(old, neu, - std::memory_order_acq_rel, - std::memory_order_acquire)) { - return {this, &n}; - } - } - } - - // Only the pool/handle can call this - void giveBack(StringBuffer&& h) noexcept { - Node* node = h.node_; - if (!node) return; // already invalid - const std::uint32_t idx = node->self_idx; - - for (;;) { - auto old = head_.load(std::memory_order_acquire); - - node->next_idx = old.idx; - - Head neu{ std::uint32_t(old.tag + 1), idx }; - if (head_.compare_exchange_weak(old, neu, - std::memory_order_acq_rel, - std::memory_order_acquire)) { - // Invalidate handle (prevents double return) - h.owner_ = nullptr; - h.node_ = nullptr; - return; - } - } - } - - private: - - void grow() { - if (head_.load(std::memory_order_acquire).idx != kEmptyIdx) return; - std::scoped_lock lk(growMutex_); - if (head_.load(std::memory_order_acquire).idx != kEmptyIdx) return; - - auto base = static_cast(nodes_.size()); - nodes_.resize(base + growBy_); - - // Init nodes and local chain - for (std::uint32_t i = 0; i < growBy_; ++i) { - std::uint32_t idx = base + i; - Node& n = nodes_[idx]; - n.self_idx = idx; - n.next_idx = (i + 1 < growBy_) ? (idx + 1) : kEmptyIdx; - } - - // Splice chain onto global head: [base .. base+grow_by_-1] - const std::uint32_t chain_head = base; - const std::uint32_t chain_tail = base + growBy_ - 1; - - for (;;) { - auto old = head_.load(std::memory_order_acquire); - - nodes_[chain_tail].next_idx = old.idx; // tail -> old head - Head neu{ std::uint32_t(old.tag + 1), chain_head }; - - if (head_.compare_exchange_weak(old, neu, - std::memory_order_acq_rel, - std::memory_order_acquire)) { - break; - } - } - } - - const std::uint32_t growBy_; - - // single 64-bit CAS - std::atomic head_; - - // only during growth - std::mutex growMutex_; - - // stable storage for nodes/strings - std::deque nodes_; - }; - using StringBuffer = StringBufferPool::StringBuffer; - class JsonLogContext { - StringBuffer messageBuffer_; + std::string messageBuffer_; detail::SimpleJsonWriter jsonWriter_; bool hasMessageParams_ = false; std::size_t messageOffset_ = 0; public: JsonLogContext() - : messageBuffer_(rentFromPool()) - , jsonWriter_(&messageBuffer_.str()) - {} + : jsonWriter_(&messageBuffer_) + { + messageBuffer_.reserve(4 * 1024); + } - StringBuffer + std::string& messageBuffer() { return messageBuffer_; } void @@ -522,7 +395,6 @@ class Journal static std::shared_mutex globalLogAttributesMutex_; static bool jsonLogsEnabled_; - static StringBufferPool messagePool_; static thread_local JsonLogContext currentJsonLogContext_; // Invariant: m_sink always points to a valid Sink @@ -533,21 +405,12 @@ class Journal std::source_location location, severities::Severity severity) const; - static StringBuffer + static std::string& formatLog(std::string const& message); public: //-------------------------------------------------------------------------- - static StringBuffer - rentFromPool() - { - return messagePool_.rent(); - } - - static void - returnStringBuffer(StringBuffer&& node) { messagePool_.giveBack(std::move(node)); } - static void enableStructuredJournal(); @@ -597,7 +460,7 @@ class Journal level is below the current threshold(). */ virtual void - write(Severity level, StringBuffer text) = 0; + write(Severity level, std::string const& text) = 0; /** Bypass filter and write text to the sink at the specified severity. * Always write the message, but maintain the same formatting as if @@ -607,7 +470,7 @@ class Journal * @param text Text to write to sink. */ virtual void - writeAlways(Severity level, StringBuffer text) = 0; + writeAlways(Severity level, std::string const& text) = 0; private: Severity thresh_; diff --git a/include/xrpl/beast/utility/WrappedSink.h b/include/xrpl/beast/utility/WrappedSink.h index 8e083c467ec..72dcf4e8f59 100644 --- a/include/xrpl/beast/utility/WrappedSink.h +++ b/include/xrpl/beast/utility/WrappedSink.h @@ -88,19 +88,17 @@ class WrappedSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, Journal::StringBuffer text) override + write(beast::severities::Severity level, std::string const& text) override { using beast::Journal; - text.str() = prefix_ + text.str(); - sink_.write(level, text); + sink_.write(level, prefix_ + text); } void - writeAlways(severities::Severity level, Journal::StringBuffer text) override + writeAlways(severities::Severity level, std::string const& text) override { using beast::Journal; - text.str() = prefix_ + text.str(); - sink_.writeAlways(level, text); + sink_.writeAlways(level, prefix_ + text); } }; diff --git a/src/libxrpl/basics/Log.cpp b/src/libxrpl/basics/Log.cpp index 7461086d6c8..8352e13e77e 100644 --- a/src/libxrpl/basics/Log.cpp +++ b/src/libxrpl/basics/Log.cpp @@ -38,12 +38,6 @@ namespace ripple { -namespace { -constexpr auto FLUSH_INTERVAL = - std::chrono::milliseconds(10); // Max delay before flush -} - - Logs::Sink::Sink( std::string const& partition, beast::severities::Severity thresh, @@ -53,7 +47,7 @@ Logs::Sink::Sink( } void -Logs::Sink::write(beast::severities::Severity level, beast::Journal::StringBuffer text) +Logs::Sink::write(beast::severities::Severity level, std::string const& text) { if (level < threshold()) return; @@ -62,7 +56,9 @@ Logs::Sink::write(beast::severities::Severity level, beast::Journal::StringBuffe } void -Logs::Sink::writeAlways(beast::severities::Severity level, beast::Journal::StringBuffer text) +Logs::Sink::writeAlways( + beast::severities::Severity level, + std::string const& text) { logs_.write(level, partition_, text, console()); } @@ -76,7 +72,7 @@ Logs::File::File() : m_stream(nullptr) bool Logs::File::isOpen() const noexcept { - return m_stream.has_value(); + return m_stream != nullptr; } bool @@ -87,16 +83,14 @@ Logs::File::open(boost::filesystem::path const& path) bool wasOpened = false; // VFALCO TODO Make this work with Unicode file paths - std::ofstream stream(path.c_str(), std::fstream::app); + std::unique_ptr stream( + new std::ofstream(path.c_str(), std::fstream::app)); - if (stream.good()) + if (stream->good()) { m_path = path; m_stream = std::move(stream); - size_t const bufsize = 256 * 1024; - static char buf[bufsize]; - m_stream->rdbuf()->pubsetbuf(buf, bufsize); wasOpened = true; } @@ -115,39 +109,31 @@ Logs::File::closeAndReopen() void Logs::File::close() { - m_stream.reset(); + m_stream = nullptr; } void -Logs::File::write(std::string const& text) +Logs::File::write(std::string_view text) { - if (m_stream.has_value()) - m_stream->write(text.data(), text.size()); + if (m_stream != nullptr) + (*m_stream) << text; +} + +void +Logs::File::writeln(std::string_view text) +{ + if (m_stream != nullptr) + { + (*m_stream) << text; + (*m_stream) << std::endl; + } } //------------------------------------------------------------------------------ Logs::Logs(beast::severities::Severity thresh) : thresh_(thresh) // default severity - , writeBuffer_( - batchBuffer_) // Initially, entire buffer is available for writing - , readBuffer_(batchBuffer_.data(), 0) // No data ready to flush initially - , stopLogThread_(false) { - logThread_ = std::thread(&Logs::logThreadWorker, this); -} - -Logs::~Logs() -{ - // Signal log thread to stop and wait for it to finish - { - stopLogThread_ = true; - } - - if (logThread_.joinable()) - logThread_.join(); - - flushBatch(); // Ensure all logs are written on shutdown } bool @@ -159,7 +145,7 @@ Logs::open(boost::filesystem::path const& pathToLogFile) beast::Journal::Sink& Logs::get(std::string const& name) { - std::lock_guard lock(sinkSetMutex_); + std::lock_guard lock(mutex_); auto const result = sinks_.emplace(name, makeSink(name, thresh_)); return *result.first->second; } @@ -170,6 +156,12 @@ Logs::operator[](std::string const& name) return get(name); } +beast::Journal +Logs::journal(std::string const& name) +{ + return beast::Journal(get(name)); +} + beast::severities::Severity Logs::threshold() const { @@ -179,7 +171,7 @@ Logs::threshold() const void Logs::threshold(beast::severities::Severity thresh) { - std::lock_guard lock(sinkSetMutex_); + std::lock_guard lock(mutex_); thresh_ = thresh; for (auto& sink : sinks_) sink.second->threshold(thresh); @@ -189,7 +181,7 @@ std::vector> Logs::partition_severities() const { std::vector> list; - std::lock_guard lock(sinkSetMutex_); + std::lock_guard lock(mutex_); list.reserve(sinks_.size()); for (auto const& [name, sink] : sinks_) list.emplace_back(name, toString(fromSeverity(sink->threshold()))); @@ -200,118 +192,28 @@ void Logs::write( beast::severities::Severity level, std::string const& partition, - beast::Journal::StringBuffer text, + std::string const& text, bool console) { std::string s; - std::string_view result = text.str(); + std::string_view result = text; if (!beast::Journal::isStructuredJournalEnabled()) { - format(s, text.str(), level, partition); - text.str() = s; - result = text.str(); - } - - // if (!silent_) - // std::cerr << result << '\n'; - - messages_.push(text); - - // Signal log thread that new messages are available - // logCondition_.notify_one(); - - // Add to batch buffer for file output - if (0) { - // std::lock_guard lock(batchMutex_); - - // Console output still immediate for responsiveness - // if (!silent_) - // std::cerr << result << '\n'; - - size_t logSize = result.size() + 1; // +1 for newline - - // If log won't fit in current write buffer, flush first - if (logSize > writeBuffer_.size()) - { - flushBatchUnsafe(); - } - - // Copy log into write buffer - std::copy(result.begin(), result.end(), writeBuffer_.begin()); - writeBuffer_[result.size()] = '\n'; - - return; - - // Update spans: expand read buffer, shrink write buffer - size_t totalUsed = readBuffer_.size() + logSize; - readBuffer_ = std::span(batchBuffer_.data(), totalUsed); - writeBuffer_ = std::span( - batchBuffer_.data() + totalUsed, batchBuffer_.size() - totalUsed); - - auto now = std::chrono::steady_clock::now(); - bool shouldFlush = (now - lastFlush_) >= FLUSH_INTERVAL; - - if (shouldFlush) - { - flushBatchUnsafe(); - lastFlush_ = now; - } + format(s, text, level, partition); + result = text; } + std::lock_guard lock(mutex_); + file_.writeln(result); // VFALCO TODO Fix console output // if (console) // out_.write_console(s); } -void -Logs::flushBatch() -{ - std::lock_guard lock(batchMutex_); - flushBatchUnsafe(); -} - -void -Logs::flushBatchUnsafe() -{ - if (readBuffer_.empty()) - return; - - // Write the read buffer contents to file in one system call - // file_.write(std::string_view{readBuffer_.data(), readBuffer_.size()}); - - // Reset spans: entire buffer available for writing, nothing to read - writeBuffer_ = std::span(batchBuffer_); - readBuffer_ = std::span(batchBuffer_.data(), 0); -} - -void -Logs::logThreadWorker() -{ - while (!stopLogThread_) - { - std::this_thread::sleep_for(FLUSH_INTERVAL); - - beast::Journal::StringBuffer buffer; - // Process all available messages - while (messages_.pop(buffer)) - { - // Also write to console if not silent - if (!silent_) - std::cerr << buffer.str() << '\n'; - - // Write to file - file_.write(buffer.str()); - - // Return node to pool for reuse - beast::Journal::returnStringBuffer(std::move(buffer)); - } - } -} - std::string Logs::rotate() { - flushBatch(); // Flush pending logs before rotating + std::lock_guard lock(mutex_); bool const wasOpened = file_.closeAndReopen(); if (wasOpened) return "The log file was closed and reopened."; @@ -428,7 +330,7 @@ Logs::fromString(std::string const& s) void Logs::format( std::string& output, - std::string_view message, + std::string const& message, beast::severities::Severity severity, std::string const& partition) { diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index f37aa05bb5e..fd3fd2546de 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -19,6 +19,8 @@ #include +#include + #include #include #include @@ -28,92 +30,9 @@ namespace beast { -namespace { - -// Fast timestamp to ISO string conversion -// Returns string like "2024-01-15T10:30:45.123Z" -std::string_view -fastTimestampToString(std::int64_t milliseconds_since_epoch) -{ - thread_local char buffer[64]; // "2024-01-15T10:30:45.123Z" - - // Precomputed lookup table for 2-digit numbers 00-99 - static constexpr char digits[200] = { - '0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', '0', '6', - '0', '7', '0', '8', '0', '9', '1', '0', '1', '1', '1', '2', '1', '3', - '1', '4', '1', '5', '1', '6', '1', '7', '1', '8', '1', '9', '2', '0', - '2', '1', '2', '2', '2', '3', '2', '4', '2', '5', '2', '6', '2', '7', - '2', '8', '2', '9', '3', '0', '3', '1', '3', '2', '3', '3', '3', '4', - '3', '5', '3', '6', '3', '7', '3', '8', '3', '9', '4', '0', '4', '1', - '4', '2', '4', '3', '4', '4', '4', '5', '4', '6', '4', '7', '4', '8', - '4', '9', '5', '0', '5', '1', '5', '2', '5', '3', '5', '4', '5', '5', - '5', '6', '5', '7', '5', '8', '5', '9', '6', '0', '6', '1', '6', '2', - '6', '3', '6', '4', '6', '5', '6', '6', '6', '7', '6', '8', '6', '9', - '7', '0', '7', '1', '7', '2', '7', '3', '7', '4', '7', '5', '7', '6', - '7', '7', '7', '8', '7', '9', '8', '0', '8', '1', '8', '2', '8', '3', - '8', '4', '8', '5', '8', '6', '8', '7', '8', '8', '8', '9', '9', '0', - '9', '1', '9', '2', '9', '3', '9', '4', '9', '5', '9', '6', '9', '7', - '9', '8', '9', '9'}; - - constexpr std::int64_t UNIX_EPOCH_DAYS = - 719468; // Days from year 0 to 1970-01-01 - - std::int64_t seconds = milliseconds_since_epoch / 1000; - int ms = milliseconds_since_epoch % 1000; - std::int64_t days = seconds / 86400 + UNIX_EPOCH_DAYS; - int sec_of_day = seconds % 86400; - - // Calculate year, month, day from days using Gregorian calendar algorithm - int era = (days >= 0 ? days : days - 146096) / 146097; - int doe = days - era * 146097; - int yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; - int year = yoe + era * 400; - int doy = doe - (365 * yoe + yoe / 4 - yoe / 100); - int mp = (5 * doy + 2) / 153; - int day = doy - (153 * mp + 2) / 5 + 1; - int month = mp + (mp < 10 ? 3 : -9); - year += (month <= 2); - - // Calculate hour, minute, second - int hour = sec_of_day / 3600; - int min = (sec_of_day % 3600) / 60; - int sec = sec_of_day % 60; - - // Format: "2024-01-15T10:30:45.123Z" - buffer[0] = '0' + year / 1000; - buffer[1] = '0' + (year / 100) % 10; - buffer[2] = '0' + (year / 10) % 10; - buffer[3] = '0' + year % 10; - buffer[4] = '-'; - buffer[5] = digits[month * 2]; - buffer[6] = digits[month * 2 + 1]; - buffer[7] = '-'; - buffer[8] = digits[day * 2]; - buffer[9] = digits[day * 2 + 1]; - buffer[10] = 'T'; - buffer[11] = digits[hour * 2]; - buffer[12] = digits[hour * 2 + 1]; - buffer[13] = ':'; - buffer[14] = digits[min * 2]; - buffer[15] = digits[min * 2 + 1]; - buffer[16] = ':'; - buffer[17] = digits[sec * 2]; - buffer[18] = digits[sec * 2 + 1]; - buffer[19] = '.'; - buffer[20] = '0' + ms / 100; - buffer[21] = '0' + (ms / 10) % 10; - buffer[22] = '0' + ms % 10; - buffer[23] = 'Z'; - - return {buffer, 24}; -} - -} // anonymous namespace - std::string Journal::globalLogAttributes_; std::shared_mutex Journal::globalLogAttributesMutex_; bool Journal::jsonLogsEnabled_ = false; -Journal::StringBufferPool Journal::messagePool_{}; thread_local Journal::JsonLogContext Journal::currentJsonLogContext_{}; //------------------------------------------------------------------------------ @@ -157,12 +76,12 @@ class NullJournalSink : public Journal::Sink } void - write(severities::Severity, Journal::StringBuffer) override + write(severities::Severity, std::string const&) override { } void - writeAlways(severities::Severity, Journal::StringBuffer) override + writeAlways(severities::Severity, std::string const&) override { } }; @@ -224,12 +143,12 @@ Journal::JsonLogContext::start( thread_local ThreadIdStringInitializer const threadId; messageOffset_ = 0; - messageBuffer_.str().clear(); - jsonWriter_ = detail::SimpleJsonWriter{&messageBuffer_.str()}; + messageBuffer_.clear(); + jsonWriter_ = detail::SimpleJsonWriter{&messageBuffer_}; if (!jsonLogsEnabled_) { - messageBuffer_.str() = journalAttributes; + messageBuffer_ = journalAttributes; return; } @@ -277,10 +196,9 @@ Journal::JsonLogContext::start( writer().writeString(severityStr); auto nowMs = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count(); + std::chrono::system_clock::now().time_since_epoch()); writer().writeKey("Tm"); - writer().writeString(fastTimestampToString(nowMs)); + writer().writeString(date::format("%Y-%b-%d %T %Z", nowMs)); writer().endObject(); @@ -290,25 +208,22 @@ Journal::JsonLogContext::start( void Journal::JsonLogContext::reuseJson() { - messageOffset_ = messageBuffer_.str().size(); + messageOffset_ = messageBuffer_.size(); } + void Journal::JsonLogContext::finish() { if (messageOffset_ != 0) { - auto buffer = rentFromPool(); - std::string_view json{messageBuffer_.str()}; - buffer.str() = json.substr(0, messageOffset_); - messageBuffer_ = buffer; + messageBuffer_.erase(messageOffset_); } else { - messageBuffer_ = rentFromPool(); + messageBuffer_.clear(); } - - messageBuffer_.str().reserve(1024 * 5); - jsonWriter_ = detail::SimpleJsonWriter{&messageBuffer_.str()}; + + jsonWriter_ = detail::SimpleJsonWriter{&messageBuffer_}; } void @@ -319,7 +234,7 @@ Journal::initMessageContext( currentJsonLogContext_.start(location, severity, name_, attributes_); } -Journal::StringBuffer +std::string& Journal::formatLog(std::string const& message) { if (!jsonLogsEnabled_) @@ -423,9 +338,8 @@ Journal::ScopedStream::~ScopedStream() if (s == "\n") s = ""; - auto messageHandle = formatLog(s); + m_sink.write(m_level, formatLog(s)); currentJsonLogContext_.finish(); - m_sink.write(m_level, messageHandle); } } diff --git a/src/test/beast/beast_Journal_test.cpp b/src/test/beast/beast_Journal_test.cpp index 43f5d1668e6..13e2726c89c 100644 --- a/src/test/beast/beast_Journal_test.cpp +++ b/src/test/beast/beast_Journal_test.cpp @@ -48,14 +48,14 @@ class Journal_test : public unit_test::suite } void - write(severities::Severity level, Journal::StringBuffer) override + write(severities::Severity level, std::string const&) override { if (level >= threshold()) ++m_count; } void - writeAlways(severities::Severity level, Journal::StringBuffer) override + writeAlways(severities::Severity level, std::string const&) override { ++m_count; } diff --git a/src/test/csf/Sim.h b/src/test/csf/Sim.h index e25d1a17f1c..0cb052305ad 100644 --- a/src/test/csf/Sim.h +++ b/src/test/csf/Sim.h @@ -49,19 +49,20 @@ class BasicSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, beast::Journal::StringBuffer text) override + write(beast::severities::Severity level, std::string const& text) override { if (level < threshold()) return; - std::cout << clock_.now().time_since_epoch().count() << " " << text.str() + std::cout << clock_.now().time_since_epoch().count() << " " << text << std::endl; } void - writeAlways(beast::severities::Severity level, beast::Journal::StringBuffer text) override + writeAlways(beast::severities::Severity level, std::string const& text) + override { - std::cout << clock_.now().time_since_epoch().count() << " " << text.str() + std::cout << clock_.now().time_since_epoch().count() << " " << text << std::endl; } }; diff --git a/src/test/jtx/CaptureLogs.h b/src/test/jtx/CaptureLogs.h index 4d3bca044b6..a8afb521c38 100644 --- a/src/test/jtx/CaptureLogs.h +++ b/src/test/jtx/CaptureLogs.h @@ -57,18 +57,19 @@ class CaptureLogs : public Logs } void - write(beast::severities::Severity level, beast::Journal::StringBuffer text) override + write(beast::severities::Severity level, std::string const& text) + override { std::lock_guard lock(strmMutex_); - strm_ << text.str(); + strm_ << text; } void - writeAlways(beast::severities::Severity level, beast::Journal::StringBuffer text) + writeAlways(beast::severities::Severity level, std::string const& text) override { std::lock_guard lock(strmMutex_); - strm_ << text.str(); + strm_ << text; } }; diff --git a/src/test/jtx/CheckMessageLogs.h b/src/test/jtx/CheckMessageLogs.h index 63c37a66197..fd3915440e1 100644 --- a/src/test/jtx/CheckMessageLogs.h +++ b/src/test/jtx/CheckMessageLogs.h @@ -45,14 +45,15 @@ class CheckMessageLogs : public Logs } void - write(beast::severities::Severity level, beast::Journal::StringBuffer text) override + write(beast::severities::Severity level, std::string const& text) + override { - if (text.str().find(owner_.msg_) != std::string::npos) + if (text.find(owner_.msg_) != std::string::npos) *owner_.pFound_ = true; } void - writeAlways(beast::severities::Severity level, beast::Journal::StringBuffer text) + writeAlways(beast::severities::Severity level, std::string const& text) override { write(level, text); diff --git a/src/test/server/Server_test.cpp b/src/test/server/Server_test.cpp index 4aa76b715ed..874558f428b 100644 --- a/src/test/server/Server_test.cpp +++ b/src/test/server/Server_test.cpp @@ -92,19 +92,20 @@ class Server_test : public beast::unit_test::suite } void - write(beast::severities::Severity level, beast::Journal::StringBuffer text) override + write(beast::severities::Severity level, std::string const& text) + override { if (level < threshold()) return; - suite_.log << text.str() << std::endl; + suite_.log << text << std::endl; } void - writeAlways(beast::severities::Severity level, beast::Journal::StringBuffer text) + writeAlways(beast::severities::Severity level, std::string const& text) override { - suite_.log << text.str() << std::endl; + suite_.log << text << std::endl; } }; diff --git a/src/test/unit_test/SuiteJournal.h b/src/test/unit_test/SuiteJournal.h index f4858cd7809..d56c297b0a2 100644 --- a/src/test/unit_test/SuiteJournal.h +++ b/src/test/unit_test/SuiteJournal.h @@ -49,14 +49,17 @@ class SuiteJournalSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, beast::Journal::StringBuffer text) override; + write(beast::severities::Severity level, std::string const& text) override; void - writeAlways(beast::severities::Severity level, beast::Journal::StringBuffer text) override; + writeAlways(beast::severities::Severity level, std::string const& text) + override; }; inline void -SuiteJournalSink::write(beast::severities::Severity level, beast::Journal::StringBuffer text) +SuiteJournalSink::write( + beast::severities::Severity level, + std::string const& text) { // Only write the string if the level at least equals the threshold. if (level >= threshold()) @@ -66,7 +69,7 @@ SuiteJournalSink::write(beast::severities::Severity level, beast::Journal::Strin inline void SuiteJournalSink::writeAlways( beast::severities::Severity level, - beast::Journal::StringBuffer text) + std::string const& text) { using namespace beast::severities; @@ -93,7 +96,7 @@ SuiteJournalSink::writeAlways( static std::mutex log_mutex; std::lock_guard lock(log_mutex); - suite_.log << s << partition_ << text.str() << std::endl; + suite_.log << s << partition_ << text << std::endl; } class SuiteJournal @@ -134,7 +137,7 @@ class StreamSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, beast::Journal::StringBuffer text) override + write(beast::severities::Severity level, std::string const& text) override { if (level < threshold()) return; @@ -142,9 +145,10 @@ class StreamSink : public beast::Journal::Sink } inline void - writeAlways(beast::severities::Severity level, beast::Journal::StringBuffer text) override + writeAlways(beast::severities::Severity level, std::string const& text) + override { - strm_ << text.str() << std::endl; + strm_ << text << std::endl; } std::stringstream const& diff --git a/src/tests/libxrpl/CMakeLists.txt b/src/tests/libxrpl/CMakeLists.txt index 73520a51282..f97283c9550 100644 --- a/src/tests/libxrpl/CMakeLists.txt +++ b/src/tests/libxrpl/CMakeLists.txt @@ -14,5 +14,3 @@ xrpl_add_test(crypto) target_link_libraries(xrpl.test.crypto PRIVATE xrpl.imports.test) xrpl_add_test(net) target_link_libraries(xrpl.test.net PRIVATE xrpl.imports.test) -xrpl_add_test(telemetry) -target_link_libraries(xrpl.test.telemetry PRIVATE xrpl.imports.test) diff --git a/src/xrpld/app/consensus/RCLConsensus.cpp b/src/xrpld/app/consensus/RCLConsensus.cpp index c43ed1fdc5e..d3485239d1d 100644 --- a/src/xrpld/app/consensus/RCLConsensus.cpp +++ b/src/xrpld/app/consensus/RCLConsensus.cpp @@ -1107,8 +1107,7 @@ RCLConsensus::startRound( RclConsensusLogger::RclConsensusLogger( char const* label, bool const validating, - beast::Journal j, - std::source_location location) + beast::Journal j) : j_(j) { if (!validating && !j.info()) @@ -1126,16 +1125,16 @@ RclConsensusLogger::~RclConsensusLogger() return; auto const duration = std::chrono::duration_cast( std::chrono::steady_clock::now() - start_); - std::stringstream outSs; outSs << header_ << "duration " << (duration.count() / 1000) << '.' << std::setw(3) << std::setfill('0') << (duration.count() % 1000) << "s. " << ss_->str(); - auto node = beast::Journal::rentFromPool(); + thread_local std::string buffer; if (beast::Journal::isStructuredJournalEnabled()) { - beast::detail::SimpleJsonWriter writer{&node.str()}; + buffer.resize(5 * 1024); + beast::detail::SimpleJsonWriter writer{&buffer}; writer.startObject(); writer.writeKey("Msg"); writer.writeString(outSs.str()); @@ -1146,9 +1145,9 @@ RclConsensusLogger::~RclConsensusLogger() } else { - node.str() = outSs.str(); + buffer = outSs.str(); } - j_.sink().writeAlways(beast::severities::kInfo, node); + j_.sink().writeAlways(beast::severities::kInfo, buffer); } } // namespace ripple diff --git a/src/xrpld/app/consensus/RCLConsensus.h b/src/xrpld/app/consensus/RCLConsensus.h index e54bae83903..38481d23633 100644 --- a/src/xrpld/app/consensus/RCLConsensus.h +++ b/src/xrpld/app/consensus/RCLConsensus.h @@ -553,14 +553,12 @@ class RclConsensusLogger beast::Journal j_; std::unique_ptr ss_; std::chrono::steady_clock::time_point start_; - std::source_location location_; public: explicit RclConsensusLogger( char const* label, bool validating, - beast::Journal j, - std::source_location location = std::source_location::current()); + beast::Journal j); ~RclConsensusLogger(); std::unique_ptr const& diff --git a/src/xrpld/app/main/Application.cpp b/src/xrpld/app/main/Application.cpp index c7cb6d07972..397261f4178 100644 --- a/src/xrpld/app/main/Application.cpp +++ b/src/xrpld/app/main/Application.cpp @@ -1644,7 +1644,6 @@ ApplicationImp::run() perfLog_->stop(); JLOG(m_journal.info()) << "Done."; - logs_->flushBatch(); } void diff --git a/src/xrpld/app/paths/Flow.cpp b/src/xrpld/app/paths/Flow.cpp index 79b3e4434bf..b0b5253ee78 100644 --- a/src/xrpld/app/paths/Flow.cpp +++ b/src/xrpld/app/paths/Flow.cpp @@ -113,17 +113,20 @@ flow( if (auto stream = j.trace()) { - stream << "\nsrc: " << src << "\ndst: " << dst + std::stringstream ss; + ss << "\nsrc: " << src << "\ndst: " << dst << "\nsrcIssue: " << srcIssue << "\ndstIssue: " << dstIssue; - stream << "\nNumStrands: " << strands.size(); + ss << "\nNumStrands: " << strands.size(); for (auto const& curStrand : strands) { - stream << "NumSteps: " << curStrand.size(); + ss << "NumSteps: " << curStrand.size(); for (auto const& step : curStrand) { - stream << '\n' << *step << '\n'; + ss << '\n' << *step << '\n'; } } + + std::move(stream) << ss.str(); } bool const srcIsXRP = isXRP(srcIssue.currency); diff --git a/src/xrpld/core/detail/Config.cpp b/src/xrpld/core/detail/Config.cpp index 7a5301389e2..7eb3a68a46d 100644 --- a/src/xrpld/core/detail/Config.cpp +++ b/src/xrpld/core/detail/Config.cpp @@ -693,8 +693,6 @@ Config::loadFromString(std::string const& fileContents) if (getSingleSection(secConfig, SECTION_LOG_STYLE, strTemp, j_)) LOG_STYLE = LogStyle::fromString(strTemp); - LOG_STYLE = LogStyle::Json; - if (getSingleSection(secConfig, SECTION_SWEEP_INTERVAL, strTemp, j_)) { SWEEP_INTERVAL = beast::lexicalCastThrow(strTemp); diff --git a/src/xrpld/overlay/detail/PeerImp.cpp b/src/xrpld/overlay/detail/PeerImp.cpp index 1d4bcad1a28..202f92c57a9 100644 --- a/src/xrpld/overlay/detail/PeerImp.cpp +++ b/src/xrpld/overlay/detail/PeerImp.cpp @@ -822,14 +822,6 @@ PeerImp::cancelTimer() noexcept } } -std::string -PeerImp::makePrefix(id_t id) -{ - std::stringstream ss; - ss << "[" << std::setfill('0') << std::setw(3) << id << "] "; - return ss.str(); -} - //------------------------------------------------------------------------------ void PeerImp::doAccept() diff --git a/src/xrpld/overlay/detail/PeerImp.h b/src/xrpld/overlay/detail/PeerImp.h index a7268fd26e1..e0e2c68a4f1 100644 --- a/src/xrpld/overlay/detail/PeerImp.h +++ b/src/xrpld/overlay/detail/PeerImp.h @@ -831,14 +831,16 @@ PeerImp::PeerImp( app_.journal("Peer"), log::attributes( log::attr("NodeID", id), - log::attr("RemoteAddress", to_string(slot->remote_endpoint())) + log::attr("RemoteAddress", to_string(slot->remote_endpoint())), + log::attr("PublicKey", toBase58(TokenType::NodePublic, publicKey)) )) , p_journal_( app_.journal("Protocol"), log::attributes( log::attr("NodeID", id), - log::attr("RemoteAddress", to_string(slot->remote_endpoint()) - ))) + log::attr("RemoteAddress", to_string(slot->remote_endpoint())), + log::attr("PublicKey", toBase58(TokenType::NodePublic, publicKey)) + )) , stream_ptr_(std::move(stream_ptr)) , socket_(stream_ptr_->next_layer().socket()) , stream_(*stream_ptr_) From 17c10de2ea152507cdc966d5d3386bbd35a81fcc Mon Sep 17 00:00:00 2001 From: JCW Date: Fri, 26 Sep 2025 15:46:10 +0100 Subject: [PATCH 85/88] Fix formatting Signed-off-by: JCW --- include/xrpl/basics/Log.h | 16 +- include/xrpl/beast/utility/Journal.h | 34 ++- src/libxrpl/beast/utility/beast_Journal.cpp | 2 +- src/tests/libxrpl/basics/log.cpp | 252 +++++++------------- src/xrpld/app/paths/Flow.cpp | 4 +- src/xrpld/overlay/detail/PeerImp.cpp | 12 +- src/xrpld/overlay/detail/PeerImp.h | 10 +- 7 files changed, 126 insertions(+), 204 deletions(-) diff --git a/include/xrpl/basics/Log.h b/include/xrpl/basics/Log.h index 2cba2ba1fb7..f48a5cf8e5d 100644 --- a/include/xrpl/basics/Log.h +++ b/include/xrpl/basics/Log.h @@ -30,8 +30,8 @@ #include #include #include -#include #include +#include namespace ripple { @@ -251,15 +251,15 @@ class Logs // Wraps a Journal::Stream to skip evaluation of // expensive argument lists if the stream is not active. #ifndef JLOG -#define JLOG_JOIN_(a,b) a##b -#define JLOG_JOIN(a,b) JLOG_JOIN_(a,b) -#define JLOG_UNIQUE(base) JLOG_JOIN(base, __LINE__) // line-based unique name +#define JLOG_JOIN_(a, b) a##b +#define JLOG_JOIN(a, b) JLOG_JOIN_(a, b) +#define JLOG_UNIQUE(base) JLOG_JOIN(base, __LINE__) // line-based unique name -#define JLOG(x) \ +#define JLOG(x) \ if (auto JLOG_UNIQUE(stream) = (x); !JLOG_UNIQUE(stream)) \ - { \ - } \ - else \ + { \ + } \ + else \ std::move(JLOG_UNIQUE(stream)) #endif diff --git a/include/xrpl/beast/utility/Journal.h b/include/xrpl/beast/utility/Journal.h index 775e7fd0910..9da59fd0b2b 100644 --- a/include/xrpl/beast/utility/Journal.h +++ b/include/xrpl/beast/utility/Journal.h @@ -22,10 +22,10 @@ #include -#include #include #include #include +#include #include #include #include @@ -96,10 +96,14 @@ class SimpleJsonWriter SimpleJsonWriter() = default; SimpleJsonWriter(SimpleJsonWriter const& other) = default; - SimpleJsonWriter& operator=(SimpleJsonWriter const& other) = default; + SimpleJsonWriter& + operator=(SimpleJsonWriter const& other) = default; std::string& - buffer() { return *buffer_; } + buffer() + { + return *buffer_; + } void startObject() const @@ -334,16 +338,18 @@ class Journal detail::SimpleJsonWriter jsonWriter_; bool hasMessageParams_ = false; std::size_t messageOffset_ = 0; - public: - JsonLogContext() - : jsonWriter_(&messageBuffer_) + public: + JsonLogContext() : jsonWriter_(&messageBuffer_) { messageBuffer_.reserve(4 * 1024); } std::string& - messageBuffer() { return messageBuffer_; } + messageBuffer() + { + return messageBuffer_; + } void startMessageParams() @@ -610,20 +616,20 @@ class Journal /** Output stream support. */ /** @{ */ ScopedStream - operator<<(std::ostream& manip(std::ostream&)) const && + operator<<(std::ostream& manip(std::ostream&)) const&& { return {*this, manip}; } template ScopedStream - operator<<(T const& t) const && + operator<<(T const& t) const&& { return {*this, t}; } ScopedStream - operator<<(std::ostream& manip(std::ostream&)) const & + operator<<(std::ostream& manip(std::ostream&)) const& { currentJsonLogContext_.reuseJson(); return {*this, manip}; @@ -631,7 +637,7 @@ class Journal template ScopedStream - operator<<(T const& t) const & + operator<<(T const& t) const& { currentJsonLogContext_.reuseJson(); return {*this, t}; @@ -956,7 +962,8 @@ setTextValue( { std::ostringstream oss; oss << value; - writer.buffer() += value;; + writer.buffer() += value; + ; } writer.buffer() += " "; } @@ -1136,7 +1143,8 @@ attributes(Pair&&... pairs) return [&](beast::detail::SimpleJsonWriter& writer) { if (beast::Journal::isStructuredJournalEnabled()) { - (detail::setJsonValue(writer, pairs.first, pairs.second, nullptr), ...); + (detail::setJsonValue(writer, pairs.first, pairs.second, nullptr), + ...); } else { diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index fd3fd2546de..e6c067d2867 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -196,7 +196,7 @@ Journal::JsonLogContext::start( writer().writeString(severityStr); auto nowMs = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()); + std::chrono::system_clock::now().time_since_epoch()); writer().writeKey("Tm"); writer().writeString(date::format("%Y-%b-%d %T %Z", nowMs)); diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index 5054d309b6c..a0bcb7ef821 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -53,14 +53,17 @@ class MockLogs : public Logs operator=(Sink const&) = delete; void - write(beast::severities::Severity level, beast::Journal::StringBuffer text) override + write( + beast::severities::Severity level, + beast::Journal::StringBuffer text) override { logs_.write(level, partition_, text, false); } void - writeAlways(beast::severities::Severity level, beast::Journal::StringBuffer text) - override + writeAlways( + beast::severities::Severity level, + beast::Journal::StringBuffer text) override { logs_.write(level, partition_, text, false); } @@ -174,12 +177,10 @@ TEST_CASE("Global attributes") CHECK(jsonLog.as_object().contains("Glb")); CHECK(jsonLog.as_object()["Glb"].is_object()); CHECK(jsonLog.as_object()["Glb"].as_object().contains("Field1")); + CHECK(jsonLog.as_object()["Glb"].as_object()["Field1"].is_string()); CHECK( - jsonLog.as_object()["Glb"].as_object()["Field1"].is_string()); - CHECK( - jsonLog.as_object()["Glb"] - .as_object()["Field1"] - .get_string() == "Value1"); + jsonLog.as_object()["Glb"].as_object()["Field1"].get_string() == + "Value1"); beast::Journal::disableStructuredJournal(); } @@ -206,20 +207,16 @@ TEST_CASE("Global attributes inheritable") CHECK(jsonLog.is_object()); CHECK(jsonLog.as_object()["Glb"].as_object().contains("Field1")); + CHECK(jsonLog.as_object()["Glb"].as_object()["Field1"].is_string()); CHECK( - jsonLog.as_object()["Glb"].as_object()["Field1"].is_string()); - CHECK( - jsonLog.as_object()["Glb"] - .as_object()["Field1"] - .get_string() == "Value1"); + jsonLog.as_object()["Glb"].as_object()["Field1"].get_string() == + "Value1"); CHECK( - jsonLog.as_object()["Jnl"] - .as_object()["Field1"] - .get_string() == "Value3"); + jsonLog.as_object()["Jnl"].as_object()["Field1"].get_string() == + "Value3"); CHECK( - jsonLog.as_object()["Jnl"] - .as_object()["Field2"] - .get_string() == "Value2"); + jsonLog.as_object()["Jnl"].as_object()["Field2"].get_string() == + "Value2"); beast::Journal::disableStructuredJournal(); } @@ -366,13 +363,16 @@ class MockSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, beast::Journal::StringBuffer text) override + write(beast::severities::Severity level, beast::Journal::StringBuffer text) + override { strm_ << text.str(); } void - writeAlways(beast::severities::Severity level, beast::Journal::StringBuffer text) override + writeAlways( + beast::severities::Severity level, + beast::Journal::StringBuffer text) override { strm_ << text.str(); } @@ -387,7 +387,8 @@ class JsonLogStreamFixture { beast::Journal::resetGlobalAttributes(); beast::Journal::enableStructuredJournal(); - j_ = beast::Journal{sink_, "Test", log::attributes(log::attr("Field1", "Value1"))}; + j_ = beast::Journal{ + sink_, "Test", log::attributes(log::attr("Field1", "Value1"))}; } ~JsonLogStreamFixture() @@ -461,9 +462,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log levels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["Mtd"] - .as_object()["Lv"] - .get_string() == + logValue.as_object()["Mtd"].as_object()["Lv"].get_string() == beast::severities::to_string(beast::severities::kTrace)); } @@ -476,9 +475,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log levels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["Mtd"] - .as_object()["Lv"] - .get_string() == + logValue.as_object()["Mtd"].as_object()["Lv"].get_string() == beast::severities::to_string(beast::severities::kDebug)); } @@ -491,9 +488,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log levels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["Mtd"] - .as_object()["Lv"] - .get_string() == + logValue.as_object()["Mtd"].as_object()["Lv"].get_string() == beast::severities::to_string(beast::severities::kInfo)); } @@ -506,9 +501,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log levels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["Mtd"] - .as_object()["Lv"] - .get_string() == + logValue.as_object()["Mtd"].as_object()["Lv"].get_string() == beast::severities::to_string(beast::severities::kWarning)); } @@ -521,9 +514,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log levels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["Mtd"] - .as_object()["Lv"] - .get_string() == + logValue.as_object()["Mtd"].as_object()["Lv"].get_string() == beast::severities::to_string(beast::severities::kError)); } @@ -536,9 +527,7 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log levels") CHECK(ec == boost::system::errc::success); CHECK( - logValue.as_object()["Mtd"] - .as_object()["Lv"] - .get_string() == + logValue.as_object()["Mtd"].as_object()["Lv"].get_string() == beast::severities::to_string(beast::severities::kFatal)); } } @@ -569,23 +558,14 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log params") CHECK(ec == boost::system::errc::success); CHECK(logValue.as_object()["Dt"].is_object()); - CHECK(logValue.as_object()["Dt"] - .as_object()["Field1"] - .is_number()); + CHECK(logValue.as_object()["Dt"].as_object()["Field1"].is_number()); + CHECK(logValue.as_object()["Dt"].as_object()["Field1"].get_int64() == 1); + CHECK(logValue.as_object()["Dt"].as_object()["Field2"].is_number()); CHECK( - logValue.as_object()["Dt"] - .as_object()["Field1"] - .get_int64() == 1); - CHECK(logValue.as_object()["Dt"] - .as_object()["Field2"] - .is_number()); - CHECK( - logValue.as_object()["Dt"] - .as_object()["Field2"] - .get_uint64() == std::numeric_limits::max()); - auto field3Val = logValue.as_object()["Dt"] - .as_object()["Field3"] - .get_double(); + logValue.as_object()["Dt"].as_object()["Field2"].get_uint64() == + std::numeric_limits::max()); + auto field3Val = + logValue.as_object()["Dt"].as_object()["Field3"].get_double(); auto difference = std::abs(field3Val - std::numbers::pi); CHECK(difference < 1e-4); CHECK(logValue.as_object()["Msg"].is_string()); @@ -606,23 +586,15 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test json log fields") CHECK(ec == boost::system::errc::success); CHECK(logValue.as_object()["Dt"].is_object()); - CHECK(logValue.as_object()["Dt"] - .as_object()["Field1"] - .is_number()); - CHECK( - logValue.as_object()["Dt"] - .as_object()["Field1"] - .get_int64() == 1); + CHECK(logValue.as_object()["Dt"].as_object()["Field1"].is_number()); + CHECK(logValue.as_object()["Dt"].as_object()["Field1"].get_int64() == 1); // UInt64 doesn't fit in Json::Value so it should be converted to a string // NOTE: We should expect it to be an int64 after we make the json library // support in64 and uint64 - CHECK(logValue.as_object()["Dt"] - .as_object()["Field2"] - .is_number()); + CHECK(logValue.as_object()["Dt"].as_object()["Field2"].is_number()); CHECK( - logValue.as_object()["Dt"] - .as_object()["Field2"] - .get_uint64() == std::numeric_limits::max()); + logValue.as_object()["Dt"].as_object()["Field2"].get_uint64() == + std::numeric_limits::max()); CHECK(logValue.as_object()["Msg"].is_string()); CHECK(logValue.as_object()["Msg"].get_string() == "Test"); } @@ -639,20 +611,12 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test journal attributes") auto logValue = boost::json::parse(stream().str(), ec); CHECK(ec == boost::system::errc::success); - CHECK(logValue.as_object()["Jnl"] - .as_object()["Field1"] - .is_string()); + CHECK(logValue.as_object()["Jnl"].as_object()["Field1"].is_string()); CHECK( - logValue.as_object()["Jnl"] - .as_object()["Field1"] - .get_string() == std::string{"Value1"}); - CHECK(logValue.as_object()["Jnl"] - .as_object()["Field2"] - .is_number()); - CHECK( - logValue.as_object()["Jnl"] - .as_object()["Field2"] - .get_int64() == 2); + logValue.as_object()["Jnl"].as_object()["Field1"].get_string() == + std::string{"Value1"}); + CHECK(logValue.as_object()["Jnl"].as_object()["Field2"].is_number()); + CHECK(logValue.as_object()["Jnl"].as_object()["Field2"].get_int64() == 2); } TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test journal attributes inheritable") @@ -668,27 +632,16 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test journal attributes inheritable") auto logValue = boost::json::parse(stream().str(), ec); CHECK(ec == boost::system::errc::success); - CHECK(logValue.as_object()["Jnl"] - .as_object()["Field1"] - .is_string()); - CHECK( - logValue.as_object()["Jnl"] - .as_object()["Field1"] - .get_string() == std::string{"Value1"}); - CHECK(logValue.as_object()["Jnl"] - .as_object()["Field3"] - .is_string()); + CHECK(logValue.as_object()["Jnl"].as_object()["Field1"].is_string()); CHECK( - logValue.as_object()["Jnl"] - .as_object()["Field3"] - .get_string() == std::string{"Value3"}); - CHECK(logValue.as_object()["Jnl"] - .as_object()["Field2"] - .is_number()); + logValue.as_object()["Jnl"].as_object()["Field1"].get_string() == + std::string{"Value1"}); + CHECK(logValue.as_object()["Jnl"].as_object()["Field3"].is_string()); CHECK( - logValue.as_object()["Jnl"] - .as_object()["Field2"] - .get_int64() == 2); + logValue.as_object()["Jnl"].as_object()["Field3"].get_string() == + std::string{"Value3"}); + CHECK(logValue.as_object()["Jnl"].as_object()["Field2"].is_number()); + CHECK(logValue.as_object()["Jnl"].as_object()["Field2"].get_int64() == 2); } TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test copying journal") @@ -706,20 +659,13 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test copying journal") auto logValue = boost::json::parse(stream().str(), ec); CHECK(ec == boost::system::errc::success); - CHECK(logValue.as_object()["Jnl"] - .as_object()["Field1"] - .is_string()); + CHECK(logValue.as_object()["Jnl"].as_object()["Field1"].is_string()); CHECK( - logValue.as_object()["Jnl"] - .as_object()["Field1"] - .get_string() == std::string{"Value1"}); - CHECK(logValue.as_object()["Jnl"] - .as_object()["Field2"] - .is_number()); + logValue.as_object()["Jnl"].as_object()["Field1"].get_string() == + std::string{"Value1"}); + CHECK(logValue.as_object()["Jnl"].as_object()["Field2"].is_number()); CHECK( - logValue.as_object()["Jnl"] - .as_object()["Field2"] - .get_int64() == 2); + logValue.as_object()["Jnl"].as_object()["Field2"].get_int64() == 2); } { stream().str(""); @@ -735,20 +681,13 @@ TEST_CASE_FIXTURE(JsonLogStreamFixture, "Test copying journal") auto logValue = boost::json::parse(stream().str(), ec); CHECK(ec == boost::system::errc::success); - CHECK(logValue.as_object()["Jnl"] - .as_object()["Field1"] - .is_string()); + CHECK(logValue.as_object()["Jnl"].as_object()["Field1"].is_string()); CHECK( - logValue.as_object()["Jnl"] - .as_object()["Field1"] - .get_string() == std::string{"Value1"}); - CHECK(logValue.as_object()["Jnl"] - .as_object()["Field2"] - .is_number()); + logValue.as_object()["Jnl"].as_object()["Field1"].get_string() == + std::string{"Value1"}); + CHECK(logValue.as_object()["Jnl"].as_object()["Field2"].is_number()); CHECK( - logValue.as_object()["Jnl"] - .as_object()["Field2"] - .get_int64() == 2); + logValue.as_object()["Jnl"].as_object()["Field2"].get_int64() == 2); } } @@ -767,28 +706,17 @@ TEST_CASE_FIXTURE( auto logValue = boost::json::parse(stream().str(), ec); CHECK(ec == boost::system::errc::success); - CHECK(logValue.as_object()["Jnl"] - .as_object()["Field1"] - .is_string()); + CHECK(logValue.as_object()["Jnl"].as_object()["Field1"].is_string()); CHECK( - logValue.as_object()["Jnl"] - .as_object()["Field1"] - .get_string() == std::string{"Value1"}); - CHECK(logValue.as_object()["Jnl"] - .as_object()["Field3"] - .is_string()); + logValue.as_object()["Jnl"].as_object()["Field1"].get_string() == + std::string{"Value1"}); + CHECK(logValue.as_object()["Jnl"].as_object()["Field3"].is_string()); CHECK( - logValue.as_object()["Jnl"] - .as_object()["Field3"] - .get_string() == std::string{"Value3"}); + logValue.as_object()["Jnl"].as_object()["Field3"].get_string() == + std::string{"Value3"}); // Field2 should be overwritten to 0 - CHECK(logValue.as_object()["Jnl"] - .as_object()["Field2"] - .is_number()); - CHECK( - logValue.as_object()["Jnl"] - .as_object()["Field2"] - .get_int64() == 2); + CHECK(logValue.as_object()["Jnl"].as_object()["Field2"].is_number()); + CHECK(logValue.as_object()["Jnl"].as_object()["Field2"].get_int64() == 2); } TEST_CASE_FIXTURE( @@ -809,20 +737,12 @@ TEST_CASE_FIXTURE( auto logValue = boost::json::parse(stream().str(), ec); CHECK(ec == boost::system::errc::success); - CHECK(logValue.as_object()["Jnl"] - .as_object()["Field1"] - .is_string()); + CHECK(logValue.as_object()["Jnl"].as_object()["Field1"].is_string()); CHECK( - logValue.as_object()["Jnl"] - .as_object()["Field1"] - .get_string() == std::string{"Value1"}); - CHECK(logValue.as_object()["Jnl"] - .as_object()["Field2"] - .is_number()); - CHECK( - logValue.as_object()["Jnl"] - .as_object()["Field2"] - .get_int64() == 2); + logValue.as_object()["Jnl"].as_object()["Field1"].get_string() == + std::string{"Value1"}); + CHECK(logValue.as_object()["Jnl"].as_object()["Field2"].is_number()); + CHECK(logValue.as_object()["Jnl"].as_object()["Field2"].get_int64() == 2); } TEST_CASE_FIXTURE( @@ -843,18 +763,10 @@ TEST_CASE_FIXTURE( auto logValue = boost::json::parse(stream().str(), ec); CHECK(ec == boost::system::errc::success); - CHECK(logValue.as_object()["Jnl"] - .as_object()["Field1"] - .is_string()); - CHECK( - logValue.as_object()["Jnl"] - .as_object()["Field1"] - .get_string() == std::string{"Value1"}); - CHECK(logValue.as_object()["Jnl"] - .as_object()["Field2"] - .is_number()); + CHECK(logValue.as_object()["Jnl"].as_object()["Field1"].is_string()); CHECK( - logValue.as_object()["Jnl"] - .as_object()["Field2"] - .get_int64() == 2); + logValue.as_object()["Jnl"].as_object()["Field1"].get_string() == + std::string{"Value1"}); + CHECK(logValue.as_object()["Jnl"].as_object()["Field2"].is_number()); + CHECK(logValue.as_object()["Jnl"].as_object()["Field2"].get_int64() == 2); } diff --git a/src/xrpld/app/paths/Flow.cpp b/src/xrpld/app/paths/Flow.cpp index b0b5253ee78..e6ce4d359ea 100644 --- a/src/xrpld/app/paths/Flow.cpp +++ b/src/xrpld/app/paths/Flow.cpp @@ -114,8 +114,8 @@ flow( if (auto stream = j.trace()) { std::stringstream ss; - ss << "\nsrc: " << src << "\ndst: " << dst - << "\nsrcIssue: " << srcIssue << "\ndstIssue: " << dstIssue; + ss << "\nsrc: " << src << "\ndst: " << dst << "\nsrcIssue: " << srcIssue + << "\ndstIssue: " << dstIssue; ss << "\nNumStrands: " << strands.size(); for (auto const& curStrand : strands) { diff --git a/src/xrpld/overlay/detail/PeerImp.cpp b/src/xrpld/overlay/detail/PeerImp.cpp index 202f92c57a9..988cbc2cae4 100644 --- a/src/xrpld/overlay/detail/PeerImp.cpp +++ b/src/xrpld/overlay/detail/PeerImp.cpp @@ -986,9 +986,9 @@ PeerImp::onReadMessage(error_code ec, std::size_t bytes_transferred) if (auto stream = journal_.trace()) { std::move(stream) << "onReadMessage: " - << (bytes_transferred > 0 - ? to_string(bytes_transferred) + " bytes" - : ""); + << (bytes_transferred > 0 + ? to_string(bytes_transferred) + " bytes" + : ""); } metrics_.recv.add_message(bytes_transferred); @@ -1068,9 +1068,9 @@ PeerImp::onWriteMessage(error_code ec, std::size_t bytes_transferred) if (auto stream = journal_.trace()) { std::move(stream) << "onWriteMessage: " - << (bytes_transferred > 0 - ? to_string(bytes_transferred) + " bytes" - : ""); + << (bytes_transferred > 0 + ? to_string(bytes_transferred) + " bytes" + : ""); } metrics_.sent.add_message(bytes_transferred); diff --git a/src/xrpld/overlay/detail/PeerImp.h b/src/xrpld/overlay/detail/PeerImp.h index e0e2c68a4f1..7e8f96368b5 100644 --- a/src/xrpld/overlay/detail/PeerImp.h +++ b/src/xrpld/overlay/detail/PeerImp.h @@ -832,15 +832,17 @@ PeerImp::PeerImp( log::attributes( log::attr("NodeID", id), log::attr("RemoteAddress", to_string(slot->remote_endpoint())), - log::attr("PublicKey", toBase58(TokenType::NodePublic, publicKey)) - )) + log::attr( + "PublicKey", + toBase58(TokenType::NodePublic, publicKey)))) , p_journal_( app_.journal("Protocol"), log::attributes( log::attr("NodeID", id), log::attr("RemoteAddress", to_string(slot->remote_endpoint())), - log::attr("PublicKey", toBase58(TokenType::NodePublic, publicKey)) - )) + log::attr( + "PublicKey", + toBase58(TokenType::NodePublic, publicKey)))) , stream_ptr_(std::move(stream_ptr)) , socket_(stream_ptr_->next_layer().socket()) , stream_(*stream_ptr_) From 373121ed78f228d5b7ce387beb03616655aee605 Mon Sep 17 00:00:00 2001 From: JCW Date: Fri, 26 Sep 2025 15:54:56 +0100 Subject: [PATCH 86/88] Fix levelisation Signed-off-by: JCW --- .github/scripts/levelization/results/ordering.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/scripts/levelization/results/ordering.txt b/.github/scripts/levelization/results/ordering.txt index 82e2bf24a6c..55df4c26724 100644 --- a/.github/scripts/levelization/results/ordering.txt +++ b/.github/scripts/levelization/results/ordering.txt @@ -138,6 +138,7 @@ test.toplevel > test.csf test.toplevel > xrpl.json test.unit_test > xrpl.basics tests.libxrpl > xrpl.basics +tests.libxrpl > xrpl.net xrpl.json > xrpl.basics xrpl.ledger > xrpl.basics xrpl.ledger > xrpl.protocol @@ -170,9 +171,6 @@ xrpld.core > xrpl.basics xrpld.core > xrpl.json xrpld.core > xrpl.net xrpld.core > xrpl.protocol -xrpld.ledger > xrpl.basics -xrpld.ledger > xrpl.json -xrpld.ledger > xrpl.protocol xrpld.nodestore > xrpl.basics xrpld.nodestore > xrpld.core xrpld.nodestore > xrpld.unity From f1482d332cb7cbec9feb614c8a7c5f6c4e2ba0db Mon Sep 17 00:00:00 2001 From: JCW Date: Fri, 26 Sep 2025 16:49:11 +0100 Subject: [PATCH 87/88] Fix errors Signed-off-by: JCW --- src/libxrpl/beast/utility/beast_Journal.cpp | 89 ++++++++++++++++++++- src/tests/libxrpl/basics/log.cpp | 22 +++-- 2 files changed, 95 insertions(+), 16 deletions(-) diff --git a/src/libxrpl/beast/utility/beast_Journal.cpp b/src/libxrpl/beast/utility/beast_Journal.cpp index e6c067d2867..1b4f9b375a7 100644 --- a/src/libxrpl/beast/utility/beast_Journal.cpp +++ b/src/libxrpl/beast/utility/beast_Journal.cpp @@ -19,8 +19,6 @@ #include -#include - #include #include #include @@ -30,6 +28,88 @@ namespace beast { +namespace { + +// Fast timestamp to ISO string conversion +// Returns string like "2024-01-15T10:30:45.123Z" +std::string_view +fastTimestampToString(std::int64_t milliseconds_since_epoch) +{ + thread_local char buffer[64]; // "2024-01-15T10:30:45.123Z" + + // Precomputed lookup table for 2-digit numbers 00-99 + static constexpr char digits[200] = { + '0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', '0', '6', + '0', '7', '0', '8', '0', '9', '1', '0', '1', '1', '1', '2', '1', '3', + '1', '4', '1', '5', '1', '6', '1', '7', '1', '8', '1', '9', '2', '0', + '2', '1', '2', '2', '2', '3', '2', '4', '2', '5', '2', '6', '2', '7', + '2', '8', '2', '9', '3', '0', '3', '1', '3', '2', '3', '3', '3', '4', + '3', '5', '3', '6', '3', '7', '3', '8', '3', '9', '4', '0', '4', '1', + '4', '2', '4', '3', '4', '4', '4', '5', '4', '6', '4', '7', '4', '8', + '4', '9', '5', '0', '5', '1', '5', '2', '5', '3', '5', '4', '5', '5', + '5', '6', '5', '7', '5', '8', '5', '9', '6', '0', '6', '1', '6', '2', + '6', '3', '6', '4', '6', '5', '6', '6', '6', '7', '6', '8', '6', '9', + '7', '0', '7', '1', '7', '2', '7', '3', '7', '4', '7', '5', '7', '6', + '7', '7', '7', '8', '7', '9', '8', '0', '8', '1', '8', '2', '8', '3', + '8', '4', '8', '5', '8', '6', '8', '7', '8', '8', '8', '9', '9', '0', + '9', '1', '9', '2', '9', '3', '9', '4', '9', '5', '9', '6', '9', '7', + '9', '8', '9', '9'}; + + constexpr std::int64_t UNIX_EPOCH_DAYS = + 719468; // Days from year 0 to 1970-01-01 + + std::int64_t seconds = milliseconds_since_epoch / 1000; + int ms = milliseconds_since_epoch % 1000; + std::int64_t days = seconds / 86400 + UNIX_EPOCH_DAYS; + int sec_of_day = seconds % 86400; + + // Calculate year, month, day from days using Gregorian calendar algorithm + int era = (days >= 0 ? days : days - 146096) / 146097; + int doe = days - era * 146097; + int yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + int year = yoe + era * 400; + int doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + int mp = (5 * doy + 2) / 153; + int day = doy - (153 * mp + 2) / 5 + 1; + int month = mp + (mp < 10 ? 3 : -9); + year += (month <= 2); + + // Calculate hour, minute, second + int hour = sec_of_day / 3600; + int min = (sec_of_day % 3600) / 60; + int sec = sec_of_day % 60; + + // Format: "2024-01-15T10:30:45.123Z" + buffer[0] = '0' + year / 1000; + buffer[1] = '0' + (year / 100) % 10; + buffer[2] = '0' + (year / 10) % 10; + buffer[3] = '0' + year % 10; + buffer[4] = '-'; + buffer[5] = digits[month * 2]; + buffer[6] = digits[month * 2 + 1]; + buffer[7] = '-'; + buffer[8] = digits[day * 2]; + buffer[9] = digits[day * 2 + 1]; + buffer[10] = 'T'; + buffer[11] = digits[hour * 2]; + buffer[12] = digits[hour * 2 + 1]; + buffer[13] = ':'; + buffer[14] = digits[min * 2]; + buffer[15] = digits[min * 2 + 1]; + buffer[16] = ':'; + buffer[17] = digits[sec * 2]; + buffer[18] = digits[sec * 2 + 1]; + buffer[19] = '.'; + buffer[20] = '0' + ms / 100; + buffer[21] = '0' + (ms / 10) % 10; + buffer[22] = '0' + ms % 10; + buffer[23] = 'Z'; + + return {buffer, 24}; +} + +} // anonymous namespace + std::string Journal::globalLogAttributes_; std::shared_mutex Journal::globalLogAttributesMutex_; bool Journal::jsonLogsEnabled_ = false; @@ -196,9 +276,10 @@ Journal::JsonLogContext::start( writer().writeString(severityStr); auto nowMs = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()); + std::chrono::system_clock::now().time_since_epoch()) + .count(); writer().writeKey("Tm"); - writer().writeString(date::format("%Y-%b-%d %T %Z", nowMs)); + writer().writeString(fastTimestampToString(nowMs)); writer().endObject(); diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index a0bcb7ef821..90755e2681b 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -55,7 +55,7 @@ class MockLogs : public Logs void write( beast::severities::Severity level, - beast::Journal::StringBuffer text) override + std::string const& text) override { logs_.write(level, partition_, text, false); } @@ -63,7 +63,7 @@ class MockLogs : public Logs void writeAlways( beast::severities::Severity level, - beast::Journal::StringBuffer text) override + std::string const& text) override { logs_.write(level, partition_, text, false); } @@ -89,19 +89,17 @@ class MockLogs : public Logs write( beast::severities::Severity level, std::string const& partition, - beast::Journal::StringBuffer text, + std::string const& text, bool console) { std::string s; - std::string_view result = text.str(); + std::string_view result = text; if (!beast::Journal::isStructuredJournalEnabled()) { - format(s, text.str(), level, partition); - text.str() = s; - result = text.str(); + format(s, text, level, partition); + result = s; } logStream_.append(result); - beast::Journal::returnStringBuffer(std::move(text)); } }; @@ -363,18 +361,18 @@ class MockSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, beast::Journal::StringBuffer text) + write(beast::severities::Severity level, std::string const& text) override { - strm_ << text.str(); + strm_ << text; } void writeAlways( beast::severities::Severity level, - beast::Journal::StringBuffer text) override + std::string const& text) override { - strm_ << text.str(); + strm_ << text; } }; From 4001748ee9d81c189c27b655c182d85a593559ca Mon Sep 17 00:00:00 2001 From: JCW Date: Mon, 29 Sep 2025 09:57:31 +0100 Subject: [PATCH 88/88] Fix formatting Signed-off-by: JCW --- src/tests/libxrpl/basics/log.cpp | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/tests/libxrpl/basics/log.cpp b/src/tests/libxrpl/basics/log.cpp index 90755e2681b..47e76d202c8 100644 --- a/src/tests/libxrpl/basics/log.cpp +++ b/src/tests/libxrpl/basics/log.cpp @@ -53,17 +53,15 @@ class MockLogs : public Logs operator=(Sink const&) = delete; void - write( - beast::severities::Severity level, - std::string const& text) override + write(beast::severities::Severity level, std::string const& text) + override { logs_.write(level, partition_, text, false); } void - writeAlways( - beast::severities::Severity level, - std::string const& text) override + writeAlways(beast::severities::Severity level, std::string const& text) + override { logs_.write(level, partition_, text, false); } @@ -361,16 +359,14 @@ class MockSink : public beast::Journal::Sink } void - write(beast::severities::Severity level, std::string const& text) - override + write(beast::severities::Severity level, std::string const& text) override { strm_ << text; } void - writeAlways( - beast::severities::Severity level, - std::string const& text) override + writeAlways(beast::severities::Severity level, std::string const& text) + override { strm_ << text; }