logging: fix dictionary logging and added pytest

CMakeLists.txt

@@ -1984,22 +1984,39 @@ elseif(CONFIG_LOG_MIPI_SYST_USE_CATALOG)
 endif()
 
 if(LOG_DICT_DB_NAME_ARG)
-  if (NOT CONFIG_LOG_DICTIONARY_DB_TARGET)
-    set(LOG_DICT_DB_ALL_TARGET ALL)
-  endif()
-  add_custom_command(
-    OUTPUT ${LOG_DICT_DB_NAME}
-    COMMAND
+  set(log_dict_gen_command
       ${PYTHON_EXECUTABLE}
       ${ZEPHYR_BASE}/scripts/logging/dictionary/database_gen.py
       ${KERNEL_ELF_NAME}
       ${LOG_DICT_DB_NAME_ARG}=${LOG_DICT_DB_NAME}
       --build-header ${PROJECT_BINARY_DIR}/include/generated/zephyr/version.h
-    WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
-    COMMENT "Generating logging dictionary database: ${LOG_DICT_DB_NAME}"
-    DEPENDS ${logical_target_for_zephyr_elf}
   )
-  add_custom_target(log_dict_db_gen ${LOG_DICT_DB_ALL_TARGET} DEPENDS ${LOG_DICT_DB_NAME})
+
+  if (NOT CONFIG_LOG_DICTIONARY_DB_TARGET)
+    # If not using a separate target for generating logging dictionary
+    # database, add the generation to post build command to make sure
+    # the database is actually being generated.
+    list(APPEND
+      post_build_commands
+      COMMAND ${CMAKE_COMMAND} -E echo "Generating logging dictionary database: ${LOG_DICT_DB_NAME}"
+      COMMAND ${log_dict_gen_command}
+    )
+    list(APPEND
+      post_build_byproducts
+      ${LOG_DICT_DB_NAME}
+    )
+  else()
+    # Seprate build target for generating logging dictionary database.
+    # This needs to be explicitly called/used to generate the database.
+    add_custom_command(
+      OUTPUT ${LOG_DICT_DB_NAME}
+      COMMAND ${log_dict_gen_command}
+      WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+      COMMENT "Generating logging dictionary database: ${LOG_DICT_DB_NAME}"
+      DEPENDS ${logical_target_for_zephyr_elf}
+    )
+    add_custom_target(log_dict_db_gen DEPENDS ${LOG_DICT_DB_NAME})
+  endif()
 endif()
 
 # Add post_build_commands to post-process the final .elf file produced by

include/zephyr/logging/log_output_dict.h

@@ -30,10 +30,10 @@ enum log_dict_output_msg_type {
  */
 struct log_dict_output_normal_msg_hdr_t {
 	uint8_t type;
-	uint32_t domain:3;
-	uint32_t level:3;
-	uint32_t package_len:10;
-	uint32_t data_len:12;
+	uint32_t domain:4;
+	uint32_t level:4;
+	uint32_t package_len:16;
+	uint32_t data_len:16;
 	uintptr_t source;
 	log_timestamp_t timestamp;
 } __packed;

scripts/logging/dictionary/dictionary_parser/__init__.py

@@ -9,6 +9,7 @@
 """
 
 from .log_parser_v1 import LogParserV1
+from .log_parser_v3 import LogParserV3
 
 
 def get_parser(database):
@@ -19,4 +20,8 @@ def get_parser(database):
     if db_ver in [1, 2]:
         return LogParserV1(database)
 
+    # DB version 3 correspond to v3 parser
+    if db_ver == 3:
+        return LogParserV3(database)
+
     return None

scripts/logging/dictionary/dictionary_parser/data_types.py

@@ -0,0 +1,186 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2024 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Contains a class to describe data types used for
+dictionary logging.
+"""
+
+import struct
+
+class DataTypes():
+    """Class regarding data types, their alignments and sizes"""
+    INT = 0
+    UINT = 1
+    LONG = 2
+    ULONG = 3
+    LONG_LONG = 4
+    ULONG_LONG = 5
+    PTR = 6
+    DOUBLE = 7
+    LONG_DOUBLE = 8
+    NUM_TYPES = 9
+
+    def __init__(self, database):
+        self.database = database
+        self.data_types = {}
+
+        if database.is_tgt_64bit():
+            self.add_data_type(self.LONG, "q")
+            self.add_data_type(self.LONG_LONG, "q")
+            self.add_data_type(self.PTR, "Q")
+        else:
+            self.add_data_type(self.LONG, "i")
+            self.add_data_type(self.LONG_LONG, "q")
+            self.add_data_type(self.PTR, "I")
+
+        self.add_data_type(self.INT, "i")
+        self.add_data_type(self.DOUBLE, "d")
+        self.add_data_type(self.LONG_DOUBLE, "d")
+
+
+    @staticmethod
+    def get_stack_min_align(arch, is_tgt_64bit):
+        '''
+        Correspond to the VA_STACK_ALIGN and VA_STACK_MIN_ALIGN
+        in cbprintf_internal.h. Note that there might be some
+	variations that is obtained via actually running through
+	the log parser.
+
+        Return a tuple where the first element is stack alignment
+        value. The second element is true if alignment needs to
+        be further refined according to data type, false if not.
+        '''
+        if arch == "arc":
+            if is_tgt_64bit:
+                need_further_align = True
+                stack_min_align = 8
+            else:
+                need_further_align = False
+                stack_min_align = 1
+
+        elif arch == "arm64":
+            need_further_align = True
+            stack_min_align = 8
+
+        elif arch == "sparc":
+            need_further_align = False
+            stack_min_align = 1
+
+        elif arch == "x86":
+            if is_tgt_64bit:
+                need_further_align = True
+                stack_min_align = 8
+            else:
+                need_further_align = False
+                stack_min_align = 1
+
+        elif arch == "riscv32e":
+            need_further_align = False
+            stack_min_align = 1
+
+        elif arch == "riscv":
+            need_further_align = True
+
+            if is_tgt_64bit:
+                stack_min_align = 8
+            else:
+                stack_min_align = 1
+
+        elif arch == "nios2":
+            need_further_align = False
+            stack_min_align = 1
+
+        else:
+            need_further_align = True
+            stack_min_align = 1
+
+        return (stack_min_align, need_further_align)
+
+
+    @staticmethod
+    def get_data_type_align(data_type, is_tgt_64bit):
+        '''
+        Get the alignment for a particular data type.
+        '''
+        if data_type == DataTypes.LONG_LONG:
+            align = 8
+        elif data_type == DataTypes.LONG:
+            if is_tgt_64bit:
+                align = 8
+            else:
+                align = 4
+        else:
+            # va_list alignment is at least a integer
+            align = 4
+
+        return align
+
+
+    def add_data_type(self, data_type, fmt):
+        """Add one data type"""
+        if self.database.is_tgt_little_endian():
+            endianness = "<"
+        else:
+            endianness = ">"
+
+        formatter = endianness + fmt
+
+        self.data_types[data_type] = {}
+        self.data_types[data_type]['fmt'] = formatter
+
+        size = struct.calcsize(formatter)
+
+        if data_type == self.LONG_DOUBLE:
+            # Python doesn't have long double but we still
+            # need to skip correct number of bytes
+            size = 16
+
+        self.data_types[data_type]['sizeof'] = size
+
+        # Might need actual number for different architectures
+        # but these seem to work fine for now.
+        if self.database.is_tgt_64bit():
+            align = 8
+        else:
+            align = 4
+
+        # 'align' is used to "jump" over an argument so it has
+        # to be at least size of the data type.
+        align = max(align, size)
+        self.data_types[data_type]['align'] = align
+
+        # 'stack_align' should correspond to VA_STACK_ALIGN
+        # in cbprintf_internal.h
+        stack_align, need_more_align = DataTypes.get_stack_min_align(
+                                        self.database.get_arch(),
+                                        self.database.is_tgt_64bit())
+
+        if need_more_align:
+            stack_align = DataTypes.get_data_type_align(data_type,
+                                                        self.database.is_tgt_64bit())
+
+        self.data_types[data_type]['stack_align'] = stack_align
+
+
+    def get_sizeof(self, data_type):
+        """Get sizeof() of a data type"""
+        return self.data_types[data_type]['sizeof']
+
+
+    def get_alignment(self, data_type):
+        """Get the alignment of a data type"""
+        return self.data_types[data_type]['align']
+
+
+    def get_stack_alignment(self, data_type):
+        """Get the stack alignment of a data type"""
+        return self.data_types[data_type]['stack_align']
+
+
+    def get_formatter(self, data_type):
+        """Get the formatter for a data type"""
+        return self.data_types[data_type]['fmt']

scripts/logging/dictionary/dictionary_parser/log_database.py

@@ -49,6 +49,9 @@
     "posix" : {
         "kconfig": "CONFIG_ARCH_POSIX",
     },
+    "riscv32e" : {
+        "kconfig": "CONFIG_RISCV_ISA_RV32E",
+    },
     "riscv" : {
         "kconfig": "CONFIG_RISCV",
     },
@@ -62,7 +65,7 @@ class LogDatabase():
     """Class of log database"""
     # Update this if database format of dictionary based logging
     # has changed
-    ZEPHYR_DICT_LOG_VER = 2
+    ZEPHYR_DICT_LOG_VER = 3
 
     LITTLE_ENDIAN = True
     BIG_ENDIAN = False

scripts/logging/dictionary/dictionary_parser/log_parser.py

@@ -9,13 +9,54 @@
 """
 
 import abc
+from colorama import Fore
+
+from .data_types import DataTypes
+
+LOG_LEVELS = [
+    ('none', Fore.WHITE),
+    ('err', Fore.RED),
+    ('wrn', Fore.YELLOW),
+    ('inf', Fore.GREEN),
+    ('dbg', Fore.BLUE)
+]
+
+def get_log_level_str_color(lvl):
+    """Convert numeric log level to string"""
+    if lvl < 0 or lvl >= len(LOG_LEVELS):
+        return ("unk", Fore.WHITE)
+
+    return LOG_LEVELS[lvl]
+
+
+def formalize_fmt_string(fmt_str):
+    """Replace unsupported formatter"""
+    new_str = fmt_str
+
+    for spec in ['d', 'i', 'o', 'u', 'x', 'X']:
+        # Python doesn't support %ll for integer specifiers, so remove extra 'l'
+        new_str = new_str.replace("%ll" + spec, "%l" + spec)
+
+        if spec in ['x', 'X']:
+            new_str = new_str.replace("%#ll" + spec, "%#l" + spec)
+
+        # Python doesn't support %hh for integer specifiers, so remove extra 'h'
+        new_str = new_str.replace("%hh" + spec, "%h" + spec)
+
+    # No %p for pointer either, so use %x
+    new_str = new_str.replace("%p", "0x%x")
+
+    return new_str
 
 
 class LogParser(abc.ABC):
     """Abstract class of log parser"""
     def __init__(self, database):
         self.database = database
 
+        self.data_types = DataTypes(self.database)
+
+
     @abc.abstractmethod
     def parse_log_data(self, logdata, debug=False):
         """Parse log data"""