modules: memfault: Add automatic sending of coredumps over LTE

Add automatic sending of coredumps over LTE:
 - Add new file memfault_lte_coredump.c that uses LTE cereg and PDN
   to determine if the device is connected to the network.

   If connected, the layer will trigger sending of a stored coredump.
   Library is implemented with retry logic and backoff.

 - Add overlay file to SLM that enables Memfault features
 - Update Memfault sample to use the new coredump feature.
 - Add missing features to Memfault sample
 - Enable assertions by default in SLM, there is no reason why it should
    be enabled as long as there is space.

Signed-off-by: Simen S. Røstad <[email protected]>

Author: simensrostad

applications/serial_lte_modem/doc/slm_description.rst

@@ -336,6 +336,9 @@ The following configuration files are provided:
   It can be customized to fit your configuration (UART, baud rate, and so on).
   By default, it sets the baud rate of the PPP UART to 1 000 000.
 
+* :file:`overlay-memfault.conf` - Configuration file that enables `Memfault`_.
+  For more information about Memfault features in |NCS|, see :ref:`mod_memfault`.
+
 * :file:`overlay-zephyr-modem.conf`, :file:`overlay-zephyr-modem-external-mcu.conf`, :file:`overlay-zephyr-modem-nrf9160dk-nrf52840.conf`, :file:`overlay-external-mcu.overlay`,  and :file:`overlay-zephyr-modem-nrf9160dk-nrf52840.overlay` - These configuration files are used when compiling SLM to turn an nRF91 Series SiP into a Zephyr-compatible standalone modem.
   See :ref:`slm_as_zephyr_modem` for more information.
 

applications/serial_lte_modem/overlay-memfault.conf

@@ -0,0 +1,28 @@
+#
+# Copyright (c) 2025 Nordic Semiconductor
+#
+# SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
+#
+
+CONFIG_MEMFAULT=y
+CONFIG_MEMFAULT_HTTP_PERIODIC_UPLOAD=y
+CONFIG_MODEM_KEY_MGMT=y
+CONFIG_MEMFAULT_LOGGING_ENABLE=y
+CONFIG_MEMFAULT_HTTP_ENABLE=y
+CONFIG_MEMFAULT_NCS_LTE_METRICS=y
+CONFIG_MEMFAULT_NCS_STACK_METRICS=y
+CONFIG_MEMFAULT_NCS_INTERNAL_FLASH_BACKED_COREDUMP=y
+CONFIG_MEMFAULT_NCS_DEVICE_ID_IMEI=y
+CONFIG_MEMFAULT_LOGGING_RAM_SIZE=4096
+CONFIG_MEMFAULT_HEAP_STATS=y
+CONFIG_MEMFAULT_HTTP_DEDICATED_WORKQUEUE_STACK_SIZE=1560
+CONFIG_MEMFAULT_COREDUMP_FULL_THREAD_STACKS=y
+CONFIG_MEMFAULT_EVENT_STORAGE_SIZE=2048
+CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED=y
+CONFIG_PDN=y
+CONFIG_LTE_LINK_CONTROL=y
+CONFIG_LTE_LC_EDRX_MODULE=y
+CONFIG_LTE_LC_PSM_MODULE=y
+
+# Memfault depends on POSIX, disable unneeded POSIX features
+CONFIG_POSIX_FILE_SYSTEM=n

applications/serial_lte_modem/prj.conf

@@ -5,6 +5,8 @@
 #
 # General config
 CONFIG_LOG=y
+CONFIG_ASSERT=y
+CONFIG_ASSERT_VERBOSE=n
 CONFIG_LOG_DEFAULT_LEVEL=3
 CONFIG_STACK_SENTINEL=y
 CONFIG_PICOLIBC_IO_FLOAT=y
@@ -118,7 +120,6 @@ CONFIG_SLM_CUSTOMER_VERSION=""
 CONFIG_SLM_EXTERNAL_XTAL=n
 
 # debug options
-#CONFIG_ASSERT=y
 #CONFIG_LOG_BUFFER_SIZE=16384
 #CONFIG_SLM_LOG_LEVEL_DBG=y
 #CONFIG_LOG_PRINTK=n

applications/serial_lte_modem/sample.yaml

@@ -21,6 +21,25 @@ tests:
     tags:
       - ci_build
       - sysbuild
+  applications.serial_lte_modem.memfault:
+    sysbuild: true
+    build_only: true
+    extra_args:
+      - EXTRA_CONF_FILE=overlay-memfault.conf
+      - CONFIG_MEMFAULT_NCS_PROJECT_KEY="dummy-key"
+    platform_allow:
+      - nrf9160dk/nrf9160/ns
+      - nrf9161dk/nrf9161/ns
+      - nrf9151dk/nrf9151/ns
+      - nrf9131ek/nrf9131/ns
+    integration_platforms:
+      - nrf9160dk/nrf9160/ns
+      - nrf9161dk/nrf9161/ns
+      - nrf9151dk/nrf9151/ns
+      - nrf9131ek/nrf9131/ns
+    tags:
+      - ci_build
+      - sysbuild
   applications.serial_lte_modem.native_tls:
     sysbuild: true
     build_only: true

applications/serial_lte_modem/src/lwm2m_carrier/slm_at_carrier.c

@@ -8,6 +8,7 @@
 #include <string.h>
 #include <time.h>
 #include <lwm2m_carrier.h>
+#include <zephyr/sys/util.h>
 #include <zephyr/logging/log.h>
 #include <modem/nrf_modem_lib.h>
 #include <modem/at_parser.h>
@@ -51,7 +52,7 @@ static void print_err(const lwm2m_carrier_event_t *evt)
 			"Connection failure",
 	};
 
-	__ASSERT(PART_OF_ARRAY(strerr[err->type]),
+	__ASSERT(PART_OF_ARRAY(strerr, strerr[err->type]),
 		 "Unhandled liblwm2m_carrier error");
 
 	LOG_ERR("LWM2M_CARRIER_EVENT_ERROR: %s, reason %d", strerr[err->type], err->value);
@@ -87,7 +88,7 @@ static void print_deferred(const lwm2m_carrier_event_t *evt)
 			"Waiting for SIM MSISDN",
 	};
 
-	__ASSERT(PART_OF_ARRAY(strdef[def->reason]),
+	__ASSERT(PART_OF_ARRAY(strdef, strdef[def->reason]),
 		 "Unhandled deferred reason");
 
 	LOG_INF("LWM2M_CARRIER_EVENT_DEFERRED: reason %s, timeout %d seconds",

applications/serial_lte_modem/src/slm_at_commands.c

@@ -81,7 +81,7 @@ enum sleep_modes {
 static struct {
 	struct k_work_delayable work;
 	uint32_t mode;
-} sleep;
+} sleep_control;
 #endif
 
 bool verify_datamode_control(uint16_t time_limit, uint16_t *time_limit_min);
@@ -130,13 +130,13 @@ static int handle_at_slmver(enum at_parser_cmd_type cmd_type, struct at_parser *
 
 static void go_sleep_wk(struct k_work *)
 {
-	if (sleep.mode == SLEEP_MODE_IDLE) {
+	if (sleep_control.mode == SLEEP_MODE_IDLE) {
 		if (slm_at_host_power_off() == 0) {
 			slm_enter_idle();
 		} else {
 			LOG_ERR("failed to power off UART");
 		}
-	} else if (sleep.mode == SLEEP_MODE_DEEP) {
+	} else if (sleep_control.mode == SLEEP_MODE_DEEP) {
 		slm_enter_sleep();
 	}
 }
@@ -148,12 +148,13 @@ static int handle_at_sleep(enum at_parser_cmd_type cmd_type, struct at_parser *p
 	int ret = -EINVAL;
 
 	if (cmd_type == AT_PARSER_CMD_TYPE_SET) {
-		ret = at_parser_num_get(parser, 1, &sleep.mode);
+		ret = at_parser_num_get(parser, 1, &sleep_control.mode);
 		if (ret) {
 			return -EINVAL;
 		}
-		if (sleep.mode == SLEEP_MODE_DEEP || sleep.mode == SLEEP_MODE_IDLE) {
-			k_work_reschedule(&sleep.work, SLM_UART_RESPONSE_DELAY);
+		if (sleep_control.mode == SLEEP_MODE_DEEP ||
+		    sleep_control.mode == SLEEP_MODE_IDLE) {
+			k_work_reschedule(&sleep_control.work, SLM_UART_RESPONSE_DELAY);
 		} else {
 			ret = -EINVAL;
 		}
@@ -376,7 +377,7 @@ int slm_at_init(void)
 	int err;
 
 #if POWER_PIN_IS_ENABLED
-	k_work_init_delayable(&sleep.work, go_sleep_wk);
+	k_work_init_delayable(&sleep_control.work, go_sleep_wk);
 #endif
 
 	err = slm_at_tcp_proxy_init();

doc/nrf/libraries/debug/memfault_ncs.rst

@@ -58,6 +58,19 @@ To have these configuration files in the include path, add the following to the
 
 .. memfault_config_files_end
 
+Automatic sending of coredumps to Memfault
+==========================================
+
+To post a stored coredump from a previous crash to Memfault upon network connection, set the :kconfig:option:`CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED` Kconfig option to ``y``.
+The option is only supported for nRF91 Series devices.
+
+The library has built-in connection awareness and tries to post the coredump to a maximum of the number set in the :kconfig:option:`CONFIG_MEMFAULT_NCS_POST_COREDUMP_RETRIES_MAX` Kconfig option, at an interval of the time set in the :kconfig:option:`CONFIG_MEMFAULT_NCS_POST_COREDUMP_RETRY_INTERVAL_SECONDS` Kconfig option between each attempt.
+If unsuccessful within the number of attempts, the library gives up.
+If at any point the network is lost during the retry process, the library waits for the device to reconnect before restarting the retry process.
+
+This feature is useful when you want to post the coredump as soon as possible after a crash and it is not desirable to wait for the next periodic upload set by :kconfig:option:`CONFIG_MEMFAULT_HTTP_PERIODIC_UPLOAD_INTERVAL_SECS`.
+Alternatively, you can manually trigger the coredump upload by calling the :c:func:`memfault_zephyr_port_post_data` function.
+You can use the :c:func:`memfault_coredump_has_valid_coredump` function to check whether a coredump is available.
 
 Configuration options in Memfault SDK
 =====================================
@@ -93,6 +106,7 @@ Configuration options in |NCS|
 
 The Kconfig options for Memfault that are defined in |NCS| provide some additional features compared to the options that are already implemented in Memfault SDK:
 
+* :kconfig:option:`CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED`
 * :kconfig:option:`CONFIG_MEMFAULT_NCS_PROJECT_KEY`
 * :kconfig:option:`CONFIG_MEMFAULT_NCS_PROVISION_CERTIFICATES`
 * :kconfig:option:`CONFIG_MEMFAULT_NCS_INTERNAL_FLASH_BACKED_COREDUMP`

doc/nrf/releases_and_maturity/releases/release-notes-changelog.rst

@@ -253,6 +253,8 @@ nRF Machine Learning (Edge Impulse)
 Serial LTE modem
 ----------------
 
+* Added an overlay :file:`overlay-memfault.conf` file to enable Memfault.
+  For more information about Memfault features in |NCS|, see :ref:`mod_memfault`.
 * Updated the application to use the :ref:`lib_downloader` library instead of the deprecated :ref:`lib_download_client` library.
 
 Thingy:53: Matter weather station
@@ -702,7 +704,9 @@ Edge Impulse integration
 Memfault integration
 --------------------
 
-|no_changes_yet_note|
+* Added a new feature to automatically post coredumps to Memfault when network connectivity is available.
+  To enable this feature, set the :kconfig:option:`CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED` Kconfig option to ``y``.
+  Only supported for nRF91 Series devices.
 
 AVSystem integration
 --------------------

modules/memfault-firmware-sdk/CMakeLists.txt

@@ -31,6 +31,10 @@ zephyr_library_sources_ifdef(
         CONFIG_MEMFAULT_NCS_BT_METRICS
         memfault_bt_metrics.c)
 
+zephyr_library_sources_ifdef(
+        CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED
+        memfault_lte_coredump.c)
+
 zephyr_library_sources_ifdef(
         CONFIG_MEMFAULT_NCS_ETB_CAPTURE
         memfault_etb_trace_capture.c)

modules/memfault-firmware-sdk/Kconfig

@@ -191,6 +191,43 @@ config MEMFAULT_NCS_IMPLEMENT_METRICS_COLLECTION
 	  Implement the Memfault 'memfault_metrics_heartbeat_collect_data()' function
 	  for the selected metrics. Disable this to override the implementation.
 
+config MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED
+	bool "Post coredump on network connected"
+	depends on PDN
+	depends on LTE_LINK_CONTROL
+	depends on SOC_SERIES_NRF91X
+	select SMF
+	select SMF_ANCESTOR_SUPPORT
+	select SMF_INITIAL_TRANSITION
+	help
+	  Post coredump to Memfault when the device is connected to LTE network.
+	  This option is only supported for nRF91 targets.
+
+config MEMFAULT_NCS_POST_COREDUMP_AFTER_INITIAL_DELAY
+	bool "Post coredump after initial delay"
+	default y
+	help
+	  Delay coredump posting to Memfault to avoid conflicts with the application's
+	  TLS connection setup that occurs typically LTE network is obtained.
+	  The delay duration is defined by CONFIG_MEMFAULT_NCS_POST_COREDUMP_RETRY_INTERVAL_SECONDS.
+
+config MEMFAULT_NCS_POST_COREDUMP_RETRIES_MAX
+	int "Max number of coredump post retries"
+	default 3
+	default 4 if MEMFAULT_NCS_POST_COREDUMP_AFTER_INITIAL_WAIT
+	help
+	  Maximum number of retries to post a coredump to Memfault.
+
+config MEMFAULT_NCS_POST_COREDUMP_RETRY_INTERVAL_SECONDS
+	int "Retry interval in seconds"
+	default 30
+	help
+	  Interval in seconds between coredump post retries.
+
+config MEMFAULT_NCS_POST_COREDUMP_THREAD_STACK_SIZE
+	int "Post coredump thread size"
+	default 512
+
 config MEMFAULT_NCS_ETB_CAPTURE
 	bool "Enable ETB trace capture"
 	depends on ETB_TRACE