Merge pull request #727 from elbeno/fixed-string-ids

✨ Add fixed string IDs to logs

Author: lukevalenty

include/log/catalog/catalog.hpp

@@ -4,7 +4,7 @@
 
 namespace sc {
 template <typename...> struct args;
-template <typename, typename T, T...> struct undefined;
+template <typename, auto, typename T, T...> struct undefined;
 
 template <typename> struct message {};
 template <typename> struct module_string {};

include/log/catalog/encoder.hpp

@@ -4,6 +4,7 @@
 #include <log/catalog/catalog.hpp>
 #include <log/log.hpp>
 #include <log/module.hpp>
+#include <log/string_id.hpp>
 
 #include <stdx/ct_string.hpp>
 #include <stdx/span.hpp>
@@ -20,24 +21,26 @@
 
 namespace logging::binary {
 namespace detail {
-template <typename S, typename... Args> constexpr static auto to_message() {
+template <typename S, auto Id, typename... Args>
+constexpr static auto to_message() {
     constexpr auto s = std::string_view{S::value};
     using char_t = typename std::remove_cv_t<decltype(s)>::value_type;
     return [&]<std::size_t... Is>(std::integer_sequence<std::size_t, Is...>) {
         return sc::message<
-            sc::undefined<sc::args<Args...>, char_t, s[Is]...>>{};
+            sc::undefined<sc::args<Args...>, Id, char_t, s[Is]...>>{};
     }(std::make_integer_sequence<std::size_t, std::size(s)>{});
 }
 
 template <stdx::ct_string S> constexpr static auto to_module() {
     constexpr auto s = std::string_view{S};
     return [&]<std::size_t... Is>(std::integer_sequence<std::size_t, Is...>) {
-        return sc::module_string<sc::undefined<void, char, s[Is]...>>{};
+        return sc::module_string<sc::undefined<void, -1, char, s[Is]...>>{};
     }(std::make_integer_sequence<std::size_t, std::size(s)>{});
 }
 
-template <typename S> struct to_message_t {
-    template <typename... Args> using fn = decltype(to_message<S, Args...>());
+template <typename S, auto Id> struct to_message_t {
+    template <typename... Args>
+    using fn = decltype(to_message<S, Id, Args...>());
 };
 } // namespace detail
 
@@ -85,7 +88,8 @@ template <typename Writer> struct log_handler {
             auto builder = get_builder(Env{});
             constexpr auto L = stdx::to_underlying(get_level(Env{}));
             using Message = typename decltype(builder)::template convert_args<
-                detail::to_message_t<decltype(fr.str)>::template fn,
+                detail::to_message_t<decltype(fr.str), logging::get_string_id(
+                                                           Env{})>::template fn,
                 std::remove_cvref_t<Args>...>;
             using Module = decltype(detail::to_module<get_module(Env{})>());
             w(builder.template build<L>(catalog<Message>(), module<Module>(),

include/log/string_id.hpp

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <log/env.hpp>
+
+#include <utility>
+
+namespace logging {
+[[maybe_unused]] constexpr inline struct get_string_id_t {
+    template <typename T>
+        requires true // more constrained
+    CONSTEVAL auto operator()(T &&t) const noexcept(
+        noexcept(std::forward<T>(t).query(std::declval<get_string_id_t>())))
+        -> decltype(std::forward<T>(t).query(*this)) {
+        return std::forward<T>(t).query(*this);
+    }
+
+    CONSTEVAL auto operator()(auto &&) const -> int { return -1; }
+} get_string_id;
+} // namespace logging

test/log/catalog1_lib.cpp

@@ -32,6 +32,7 @@ auto log_one_64bit_rt_arg() -> void;
 auto log_one_formatted_rt_arg() -> void;
 auto log_with_non_default_module_id() -> void;
 auto log_with_fixed_module_id() -> void;
+auto log_with_fixed_string_id() -> void;
 
 auto log_zero_args() -> void {
     auto cfg = logging::binary::config{test_log_args_destination{}};
@@ -81,3 +82,12 @@ auto log_with_fixed_module_id() -> void {
             stdx::ct_format<"Fixed ModuleID string with {} placeholder">(1));
     }
 }
+
+auto log_with_fixed_string_id() -> void {
+    CIB_WITH_LOG_ENV(logging::get_level, logging::level::TRACE,
+                     logging::get_string_id, 1337) {
+        auto cfg = logging::binary::config{test_log_args_destination{}};
+        cfg.logger.log_msg<cib_log_env_t>(
+            stdx::ct_format<"Fixed StringID string">());
+    }
+}

test/log/catalog_app.cpp

@@ -19,6 +19,7 @@ extern auto log_two_rt_args() -> void;
 extern auto log_rt_enum_arg() -> void;
 extern auto log_with_non_default_module_id() -> void;
 extern auto log_with_fixed_module_id() -> void;
+extern auto log_with_fixed_string_id() -> void;
 
 TEST_CASE("log zero arguments", "[catalog]") {
     test_critical_section::count = 0;
@@ -99,3 +100,13 @@ TEST_CASE("log with fixed module id", "[catalog]") {
     CHECK((last_header & expected_static) == expected_static);
     CHECK((last_header & ~expected_static) == (17u << 16u));
 }
+
+TEST_CASE("log with fixed string id", "[catalog]") {
+    test_critical_section::count = 0;
+    log_calls = 0;
+    log_with_fixed_string_id();
+    CHECK(test_critical_section::count == 2);
+    CHECK(log_calls == 1);
+    // string ID 1337 is fixed by environment
+    CHECK(last_header == ((1337u << 4u) | 1u));
+}

tools/gen_str_catalog.py

@@ -32,15 +32,16 @@ def split_args(s: str) -> list[str]:
 
 
 string_re = re.compile(
-    r"sc::message<sc::undefined<sc::args<(.*)>, char, (.*)>\s*>"
+    r"sc::message<sc::undefined<sc::args<(.*)>, (.*), char, (.*)>\s*>"
 )
 
 
 def extract_string_id(line_m):
     catalog_type = line_m.group(1)
     string_m = string_re.match(line_m.group(3))
     arg_tuple = string_m.group(1)
-    string_tuple = string_m.group(2).replace("(char)", "")
+    string_id = int(string_m.group(2))
+    string_tuple = string_m.group(3).replace("(char)", "")
     string_value = "".join((chr(int(c)) for c in re.split(r"\s*,\s*", string_tuple)))
     args = split_args(arg_tuple)
 
@@ -51,11 +52,12 @@ def extract_string_id(line_m):
             type="flow" if string_value.startswith("flow.") else "msg",
             arg_types=args,
             arg_count=len(args),
+            id=string_id
         ),
     )
 
 
-module_re = re.compile(r"sc::module_string<sc::undefined<void, char, (.*)>\s?>")
+module_re = re.compile(r"sc::module_string<sc::undefined<void, -1, char, (.*)>\s?>")
 
 
 def module_string(module) -> str:
@@ -110,8 +112,18 @@ def get_id(stable_ids, key_fn, gen, obj):
         else:
             return next(gen)
 
+    unique_strings = {i[0][0]: i for i in strings}.values()
+
     stable_msg_ids, stable_module_ids = stable_ids
 
+    for _, msg in filter(lambda u: u[1]["id"] != -1, unique_strings):
+        stable_msg_ids[stable_msg_key(msg)] = msg["id"]
+
+    if len(stable_msg_ids) != len(set(stable_msg_ids.values())):
+        import collections
+        dupes = [item for item, count in collections.Counter(stable_msg_ids.values()).items() if count > 1]
+        raise Exception(f"Duplicate string IDs found: {dupes}.")
+
     old_msg_ids = set(stable_msg_ids.values())
     msg_id_gen = itertools.filterfalse(old_msg_ids.__contains__, itertools.count(0))
     get_msg_id = partial(get_id, stable_msg_ids, stable_msg_key, msg_id_gen)
@@ -122,7 +134,6 @@ def get_id(stable_ids, key_fn, gen, obj):
     )
     get_module_id = partial(get_id, stable_module_ids, stable_module_key, module_id_gen)
 
-    unique_strings = {i[0][0]: i for i in strings}.values()
     return (
         {m: {"string": module_string(m), "id": get_module_id(m)} for m in sorted(set(modules))},
         {item[0]: {**item[1], "id": get_msg_id(item[1])} for item in unique_strings},
@@ -144,7 +155,7 @@ def make_cpp_module_defn(m: str, text: str, n: int) -> str:
     return f"""/*
     "{text}"
  */
-template<> unsigned int module<sc::module_string<sc::undefined<void, char, {m}>>>() {{
+template<> unsigned int module<sc::module_string<sc::undefined<void, -1, char, {m}>>>() {{
     return {n};
 }}"""