Memfault Firmware SDK 0.27.0 (Build 325530)

Author: Memfault Inc

CHANGES.md

@@ -1,3 +1,26 @@
+### Changes between Memfault SDK 0.27.0 and SDK 0.26.1 - Oct 5, 2021
+
+#### :chart_with_upwards_trend: Improvements
+
+- Added support for using [compact logs](https://mflt.io/compact-logs) with the
+  Memfault [log subsystem](https://mflt.io/logging).
+- Added port for mynewt RTOS to Memfault SDK. (Huge thanks to @t3zeng for the
+  help here!) See
+  [sdk/embedded/ports/mynewt](sdk/embedded/ports/mynewt/README.md) for more
+  details.
+- Added support for
+  [Zephyr 2.7](https://docs.zephyrproject.org/latest/releases/release-notes-2.7.html)
+  release.
+
+#### :house: Internal
+
+- Fixed a missing symbol linker error for `memfault_fault_handler` that could
+  arise when compiling with `-flto`.
+- Fixed a compiler error in `memfault_fault_handling_arm.c` that arose when
+  using certain versions of the Clang compiler.
+- Cleaned up python scripts after enabling additional PEP8 naming convention
+  linters.
+
 ### Changes between Memfault SDK 0.26.1 and SDK 0.26.0 - Sept 20, 2021
 
 #### :house: Internal

VERSION

@@ -1,2 +1,2 @@
-BUILD ID: 314362
-GIT COMMIT: 8ea624d52
+BUILD ID: 325530
+GIT COMMIT: d622ced50

components/core/src/memfault_log.c

@@ -24,6 +24,7 @@
 #include "memfault/core/platform/overrides.h"
 #include "memfault/util/circular_buffer.h"
 #include "memfault/util/crc16_ccitt.h"
+#include "memfault/core/compact_log_serializer.h"
 
 #include "memfault/config.h"
 
@@ -93,12 +94,6 @@ bool memfault_log_get_regions(sMemfaultLogRegions *regions) {
   return true;
 }
 
-typedef enum {
-  kMemfaultLogRecordType_Preformatted = 0,
-
-  kMemfaultLogRecordType_NumTypes,
-} eMemfaultLogRecordType;
-
 static uint8_t prv_build_header(eMemfaultPlatformLogLevel level, eMemfaultLogRecordType type) {
   MEMFAULT_STATIC_ASSERT(kMemfaultPlatformLogLevel_NumLevels <= 8,
                          "Number of log levels exceed max number that log module can track");
@@ -223,6 +218,7 @@ static bool prv_read_log_iter_callback(sMfltLogIterator *iter) {
 
   ctx->log->msg[iter->entry.len] = '\0';
   ctx->log->level = memfault_log_get_level_from_hdr(iter->entry.hdr);
+  ctx->log->type = memfault_log_get_type_from_hdr(iter->entry.hdr);
   ctx->log->msg_len = iter->entry.len;
   ctx->has_log = true;
   return false;
@@ -234,6 +230,7 @@ static bool prv_read_log(sMemfaultLog *log) {
     const int rv = snprintf(log->msg, sizeof(log->msg), "... %d messages dropped ...",
                                  (int)s_memfault_ram_logger.dropped_msg_count);
     log->msg_len = (rv <= 0)  ? 0 : MEMFAULT_MIN((uint32_t)rv, sizeof(log->msg) - 1);
+    log->type = kMemfaultLogRecordType_Preformatted;
     s_memfault_ram_logger.dropped_msg_count = 0;
     return true;
   }
@@ -300,6 +297,7 @@ void memfault_vlog_save(eMemfaultPlatformLogLevel level, const char *fmt, va_lis
   if (bytes_written >= available_space) {
     bytes_written = available_space - 1;
   }
+
   memfault_log_save_preformatted(level, log_buf, bytes_written);
 }
 
@@ -310,8 +308,10 @@ void memfault_log_save(eMemfaultPlatformLogLevel level, const char *fmt, ...) {
   va_end(args);
 }
 
-void memfault_log_save_preformatted(eMemfaultPlatformLogLevel level,
-                                    const char *log, size_t log_len) {
+static void prv_log_save(eMemfaultPlatformLogLevel level,
+                         const void *log, size_t log_len,
+                         eMemfaultLogRecordType log_type) {
+
   if (!prv_should_log(level)) {
     return;
   }
@@ -326,7 +326,7 @@ void memfault_log_save_preformatted(eMemfaultPlatformLogLevel level,
     if (space_free) {
         sMfltRamLogEntry entry = {
           .len = (uint8_t)truncated_log_len,
-          .hdr = prv_build_header(level, kMemfaultLogRecordType_Preformatted),
+          .hdr = prv_build_header(level, log_type),
         };
         memfault_circular_buffer_write(circ_bufp, &entry, sizeof(entry));
         memfault_circular_buffer_write(circ_bufp, log, truncated_log_len);
@@ -340,6 +340,41 @@ void memfault_log_save_preformatted(eMemfaultPlatformLogLevel level,
   }
 }
 
+#if MEMFAULT_COMPACT_LOG_ENABLE
+
+static void prv_fill_compact_log_cb(void *ctx, uint32_t offset, const void *buf, size_t buf_len) {
+  uint8_t *log = (uint8_t *)ctx;
+  memcpy(&log[offset], buf, buf_len);
+}
+
+void memfault_compact_log_save(eMemfaultPlatformLogLevel level, uint32_t log_id,
+                               uint32_t compressed_fmt, ...) {
+  char log_buf[MEMFAULT_LOG_MAX_LINE_SAVE_LEN + 1];
+
+  sMemfaultCborEncoder encoder;
+  memfault_cbor_encoder_init(&encoder, prv_fill_compact_log_cb, log_buf, sizeof(log_buf));
+
+  va_list args;
+  va_start(args, compressed_fmt);
+  bool success = memfault_vlog_compact_serialize(&encoder, log_id, compressed_fmt, args);
+  va_end(args);
+
+  if (!success) {
+    return;
+  }
+
+  const size_t bytes_written = memfault_cbor_encoder_deinit(&encoder);
+  prv_log_save(level, log_buf, bytes_written, kMemfaultLogRecordType_Compact);
+}
+
+#endif /* MEMFAULT_COMPACT_LOG_ENABLE */
+
+
+void memfault_log_save_preformatted(eMemfaultPlatformLogLevel level,
+                                    const char *log, size_t log_len) {
+  prv_log_save(level, log, log_len, kMemfaultLogRecordType_Preformatted);
+}
+
 bool memfault_log_boot(void *storage_buffer, size_t buffer_len) {
   if (storage_buffer == NULL || buffer_len == 0 || s_memfault_ram_logger.enabled) {
     return false;

components/core/src/memfault_log_data_source.c

@@ -96,15 +96,29 @@ typedef struct {
 static bool prv_copy_msg_callback(sMfltLogIterator *iter, MEMFAULT_UNUSED size_t offset,
                                   const char *buf, size_t buf_len) {
   sMfltLogEncodingCtx *const ctx = (sMfltLogEncodingCtx *)iter->user_ctx;
-  return memfault_cbor_encode_string_add(&ctx->encoder, buf, buf_len);
+  return memfault_cbor_join(&ctx->encoder, buf, buf_len);
 }
 
 static bool prv_encode_current_log(sMemfaultCborEncoder *encoder, sMfltLogIterator *iter) {
-  return (
-    memfault_cbor_encode_unsigned_integer(encoder, memfault_log_get_level_from_hdr(iter->entry.hdr)) &&
-    memfault_cbor_encode_string_begin(encoder, iter->entry.len) &&
-    memfault_log_iter_copy_msg(iter, prv_copy_msg_callback)
-  );
+
+  if (!memfault_cbor_encode_unsigned_integer(encoder,
+                                             memfault_log_get_level_from_hdr(iter->entry.hdr))) {
+    return false;
+  }
+
+  eMemfaultLogRecordType type = memfault_log_get_type_from_hdr(iter->entry.hdr);
+  bool success;
+
+  // Note: We encode "preformatted" logs (i.e logs that have run through printf) as cbor text
+  // string and "compact" logs as a cbor byte array so we can differentiate between the two while
+  // decoding
+  if (type == kMemfaultLogRecordType_Preformatted) {
+    success = memfault_cbor_encode_string_begin(encoder, iter->entry.len);
+  } else { // kMemfaultLogRecordType_Compact
+    success = memfault_cbor_encode_byte_string_begin(encoder, iter->entry.len);
+  }
+
+  return (success && memfault_log_iter_copy_msg(iter, prv_copy_msg_callback));
 }
 
 static bool prv_log_iterate_encode_callback(sMfltLogIterator *iter) {

components/core/src/memfault_log_private.h

@@ -15,6 +15,7 @@
 #include <stdint.h>
 
 #include "memfault/core/compiler.h"
+#include "memfault/core/log.h"
 #include "memfault/core/platform/debug_log.h"
 
 #ifdef __cplusplus
@@ -31,7 +32,7 @@ extern "C" {
 //  r = read (1 if the message has been read, 0 otherwise)
 //  s = sent (1 if the message has been sent, 0 otherwise)
 //  x = rsvd
-//  t = type (0 = formatted log)
+//  t = type (0 = formatted log, 1 = compact log)
 //  l = log level (eMemfaultPlatformLogLevel)
 
 #define MEMFAULT_LOG_HDR_LEVEL_POS  0
@@ -45,6 +46,10 @@ static inline eMemfaultPlatformLogLevel memfault_log_get_level_from_hdr(uint8_t
   return (eMemfaultPlatformLogLevel)((hdr & MEMFAULT_LOG_HDR_LEVEL_MASK) >> MEMFAULT_LOG_HDR_LEVEL_POS);
 }
 
+static inline eMemfaultLogRecordType memfault_log_get_type_from_hdr(uint8_t hdr) {
+  return (eMemfaultLogRecordType)((hdr & MEMFAULT_LOG_HDR_TYPE_MASK) >> MEMFAULT_LOG_HDR_TYPE_POS);
+}
+
 typedef MEMFAULT_PACKED_STRUCT {
   // data about the message stored (details below)
   uint8_t hdr;

components/include/memfault/core/log.h

@@ -24,8 +24,11 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#include "memfault/config.h"
+#include "memfault/core/compact_log_compile_time_checks.h"
+#include "memfault/core/compact_log_helpers.h"
 #include "memfault/core/compiler.h"
-#include "memfault/core/platform/debug_log.h" // For eMemfaultPlatformLogLevel
+#include "memfault/core/platform/debug_log.h"  // For eMemfaultPlatformLogLevel
 
 #ifdef __cplusplus
 extern "C" {
@@ -65,6 +68,30 @@ void memfault_log_set_min_save_level(eMemfaultPlatformLogLevel min_log_level);
 //! } while (0)
 #define MEMFAULT_LOG_SAVE(_level, ...) memfault_log_save(_level, __VA_ARGS__)
 
+#if MEMFAULT_COMPACT_LOG_ENABLE
+
+//! Same as MEMFAULT_LOG_SAVE except logs use Memfault's "compact" log strategy which offloads
+//! formatting to the Memfault cloud to reduce on device codespace and cpu consumption. See
+//! https://mflt.io/compact-logs for more details.
+#define MEMFAULT_COMPACT_LOG_SAVE(level, format, ...)                   \
+  do {                                                                  \
+    MEMFAULT_LOGGING_RUN_COMPILE_TIME_CHECKS(format, ## __VA_ARGS__);   \
+    MEMFAULT_LOG_FMT_ELF_SECTION_ENTRY(format, ## __VA_ARGS__);         \
+    memfault_compact_log_save(level,                                    \
+                              MEMFAULT_LOG_FMT_ELF_SECTION_ENTRY_PTR,   \
+                              MFLT_GET_COMPRESSED_LOG_FMT(__VA_ARGS__), \
+                              ## __VA_ARGS__);                          \
+  } while (0)
+
+
+//! Serializes the provided compact log and saves it to backing storage
+//!
+//! @note: Should only be called via MEMFAULT_COMPACT_LOG_SAVE macro
+void memfault_compact_log_save(eMemfaultPlatformLogLevel level, uint32_t log_id,
+                               uint32_t compressed_fmt, ...);
+
+#endif /* MEMFAULT_COMPACT_LOG_ENABLE */
+
 //! Function which can be called to save a log after it has been formatted
 //!
 //! Typically a user should be able to use the MEMFAULT_LOG_SAVE macro but if your platform does
@@ -81,15 +108,22 @@ void memfault_log_set_min_save_level(eMemfaultPlatformLogLevel min_log_level);
 void memfault_log_save_preformatted(eMemfaultPlatformLogLevel level, const char *log,
                                     size_t log_len);
 
-//! Maximum length a log record can occupy
-#define MEMFAULT_LOG_MAX_LINE_SAVE_LEN 128
+typedef enum {
+  kMemfaultLogRecordType_Preformatted = 0,
+  kMemfaultLogRecordType_Compact = 1,
+  kMemfaultLogRecordType_NumTypes,
+} eMemfaultLogRecordType;
 
 typedef struct {
   // the level of the message
   eMemfaultPlatformLogLevel level;
+  // the log returned is a binary "compact log"
+  // See https://mflt.io/compact-logs for more details
+  eMemfaultLogRecordType type;
   // the length of the msg (not including NUL character)
   uint32_t msg_len;
-  // the message to print which will always be NUL terminated
+  // the message to print which will always be NUL terminated when a preformatted log is returned
+  // (so it is always safe to call printf without copying the log into another buffer yourself)
   char msg[MEMFAULT_LOG_MAX_LINE_SAVE_LEN + 1 /* '\0' */];
 } sMemfaultLog;
 
@@ -116,7 +150,7 @@ typedef struct {
 //!   // .. RTOS code to wait for log read event ..
 //!   sMemfaultLog log = { 0 };
 //!   const bool log_found = memfault_log_read(&log);
-//!   if (log_found) {
+//!   if (log_found && (log.type == kMemfaultLogRecordType_Preformatted)) {
 //!       my_platform_uart_println(log.level, log, log.msg_len);
 //!   }
 bool memfault_log_read(sMemfaultLog *log);

components/include/memfault/default_config.h

@@ -94,8 +94,15 @@ extern "C" {
 #define MEMFAULT_LOG_DATA_SOURCE_ENABLED 1
 #endif
 
-// Shouldn't typically be needed but allows for persisting of MEMFAULT_LOG_*'s
-// to be disabled via a CFLAG: CFLAGS += -DMEMFAULT_SDK_LOG_SAVE_DISABLE=1
+//! Maximum length a log record can occupy
+//!
+//! Structs holding this log may be allocated on the stack so care should be taken
+//! to make the size is not to large to blow through the allocated space for the stack.
+#ifndef MEMFAULT_LOG_MAX_LINE_SAVE_LEN
+#define MEMFAULT_LOG_MAX_LINE_SAVE_LEN 128
+#endif
+
+//! Control whether or automatic persisting of MEMFAULT_LOG_*'s is enabled
 #ifndef MEMFAULT_SDK_LOG_SAVE_DISABLE
 #define MEMFAULT_SDK_LOG_SAVE_DISABLE 0
 #endif

components/include/memfault/util/cbor.h

@@ -123,24 +123,20 @@ bool memfault_cbor_encode_string(sMemfaultCborEncoder *encoder, const char *str)
 //!
 //! @return true on success, false otherwise
 //!
-//! @note Use one or more calls to memfault_cbor_encode_string_add() to write the contents
+//! @note Use one or more calls to memfault_cbor_join() to write the contents
 //! of the string.
 bool memfault_cbor_encode_string_begin(sMemfaultCborEncoder *encoder, size_t str_len);
 
-//! Called to encode the given C string to the string started with
-//! memfault_cbor_encode_string_begin().
-//!
-//! @note Can be called one or multiple times after calling memfault_cbor_encode_string_begin().
-//! It is the responsibililty of the caller to ensure that the total number of bytes added matches
-//! the str_len that was passed to memfault_cbor_encode_string_begin().
-//! @note It is not expected to add the NULL terminator.
+//! Called to start the encoding of an arbitrary binary payload
 //!
 //! @param encoder The encoder context to use
-//! @param str The string to add
-//! @param len The number of bytes to add from str
+//! @param buf_len The length of the binary payload to store in bytes, excluding NULL terminator.
 //!
 //! @return true on success, false otherwise
-bool memfault_cbor_encode_string_add(sMemfaultCborEncoder *encoder, const char *str, size_t len);
+//!
+//! @note Use one or more calls to memfault_cbor_join() to write the contents
+//! of the string.
+bool memfault_cbor_encode_byte_string_begin(sMemfaultCborEncoder *encoder, size_t bin_len);
 
 //! Encodes a IEEE 754 double-precision float that is packed in a uint64_t
 //!

components/include/memfault/version.h

@@ -19,7 +19,7 @@ typedef struct {
   uint8_t patch;
 } sMfltSdkVersion;
 
-#define MEMFAULT_SDK_VERSION   { .major = 0, .minor = 26, .patch = 1 }
+#define MEMFAULT_SDK_VERSION   { .major = 0, .minor = 27, .patch = 0 }
 
 #ifdef __cplusplus
 }

components/panics/src/memfault_fault_handling_arm.c

@@ -109,6 +109,7 @@ void memfault_platform_fault_handler(MEMFAULT_UNUSED const sMfltRegState *regs,
 }
 #endif /* MEMFAULT_PLATFORM_FAULT_HANDLER_CUSTOM */
 
+MEMFAULT_USED
 void memfault_fault_handler(const sMfltRegState *regs, eMemfaultRebootReason reason) {
   memfault_platform_fault_handler(regs, reason);
 
@@ -365,10 +366,10 @@ void MEMFAULT_EXC_HANDLER_WATCHDOG(void) {
       "mrsne r3, psp \n"                         \
       "push {r3-r11, lr} \n"                     \
       "mov r0, sp \n"                            \
-      "ldr r1, =%0 \n"                           \
+      "ldr r1, =%c0 \n"                          \
       "b memfault_fault_handler \n"              \
       :                                          \
-      : "i" (_x)                                 \
+      : "i" ((uint16_t)_x)                       \
    )
 #else
 #define MEMFAULT_HARDFAULT_HANDLING_ASM(_x)      \