[SD] Implement buffered JSON stream writes to sd card, refactoring along logging, basic file creation implementaiton

Author: brentru

platformio.ini

@@ -79,6 +79,7 @@ lib_deps =
     https://github.com/adafruit/WiFiNINA.git
     https://github.com/Starmbi/hp_BH1750.git
     https://github.com/adafruit/RTClib.git
+    https://github.com/bblanchon/ArduinoStreamUtils.git
 
 ; Common build environment for ESP32 platform
 [common:esp32]

src/Wippersnapper_V2.cpp

@@ -1094,7 +1094,11 @@ void Wippersnapper_V2::pingBrokerV2() {
     @brief    Feeds the WDT to prevent hardware reset.
 */
 /*******************************************************/
-void Wippersnapper_V2::feedWDTV2() { Watchdog.reset(); }
+void Wippersnapper_V2::feedWDTV2() {
+    // TODO: This is a temporary fix for watchdog.reset() not firing
+    // Watchdog.reset();
+    esp_task_wdt_reset();
+}
 
 /********************************************************/
 /*!
@@ -1264,10 +1268,10 @@ void Wippersnapper_V2::connectV2() {
 */
 /**************************************************************************/
 ws_status_t Wippersnapper_V2::runV2() {
+  WsV2.feedWDTV2();
   if (!WsV2._sdCardV2->mode_offline) {
     // Handle networking functions
     runNetFSMV2();
-    WsV2.feedWDTV2();
     pingBrokerV2();
     // Process all incoming packets from Wippersnapper_V2 MQTT Broker
     WsV2._mqttV2->processPackets(10);

src/Wippersnapper_demo.ino.cpp

@@ -406,15 +406,13 @@ bool ws_sdcard::parseConfigFile() {
     return false;
   }
 
-  // Configure the status LED
-  float brightness = exportedFromDevice["statusLEDBrightness"];
-  setStatusLEDBrightness(brightness);
-
   // Mock the check-in process using the JSON's values
   CheckIn(exportedFromDevice["totalGPIOPins"],
           exportedFromDevice["totalAnalogPins"],
           exportedFromDevice["referenceVoltage"]);
 
+  setStatusLEDBrightness(exportedFromDevice["statusLEDBrightness"]);
+
   // Initialize and configure RTC
   const char *json_rtc = exportedFromDevice["rtc"];
   if (json_rtc == nullptr) {
@@ -427,6 +425,23 @@ bool ws_sdcard::parseConfigFile() {
     return false;
   }
 
+// Create new logging file on device from the RTC's timestamp
+#ifndef OFFLINE_MODE_DEBUG
+  // TODO: Refactor this out into a func
+  // TODO: Implement a counter within the log funcs to track # of lines in the
+  // file and implement a MAX_LINE cutoff
+  String logFilename = "log_" + String(GetTimestamp()) + ".json";
+  _log_filename = logFilename.c_str();
+  File32 file;
+  if (!file.open(_log_filename, FILE_WRITE)) {
+    WS_DEBUG_PRINTLN(
+        "[SD] FATAL - Failed to create initial logging file on SD card!");
+    return false;
+  }
+  WS_DEBUG_PRINT("[SD] Created new log file on SD card: ");
+  WS_DEBUG_PRINTLN(_log_filename);
+#endif
+
   // Parse the "components" array into a JsonObject
   JsonArray components_ar = doc["components"].as<JsonArray>();
   if (components_ar.isNull()) {
@@ -522,7 +537,6 @@ bool ws_sdcard::parseConfigFile() {
 */
 /**************************************************************************/
 uint32_t ws_sdcard::GetTimestamp() {
-  // Obtain RTC timestamp (TODO - refactor this out)
   DateTime now;
   if (_rtc_ds3231 != nullptr)
     now = _rtc_ds3231->now();
@@ -531,7 +545,7 @@ uint32_t ws_sdcard::GetTimestamp() {
   else if (_rtc_pcf8523 != nullptr)
     now = _rtc_pcf8523->now();
   else {
-    // TODO! implement software millis() version of now() and unixtime()
+    now = _rtc_soft->now();
   }
 
   if (_wokwi_runner)
@@ -629,6 +643,30 @@ const char *SensorTypeToString(wippersnapper_sensor_SensorType sensorType) {
   }
 }
 
+void ws_sdcard::BuildJSONDoc(JsonDocument &doc, const char *pin, float value,
+                             wippersnapper_sensor_SensorType read_type) {
+  doc["timestamp"] = GetTimestamp();
+  doc["pin"] = pin;
+  doc["value"] = value;
+  doc["si_unit"] = SensorTypeToString(read_type);
+}
+
+void ws_sdcard::BuildJSONDoc(JsonDocument &doc, const char *pin, uint16_t value,
+                             wippersnapper_sensor_SensorType read_type) {
+  doc["timestamp"] = GetTimestamp();
+  doc["pin"] = pin;
+  doc["value"] = value;
+  doc["si_unit"] = SensorTypeToString(read_type);
+}
+
+void ws_sdcard::BuildJSONDoc(JsonDocument &doc, const char *pin, bool value,
+                             wippersnapper_sensor_SensorType read_type) {
+  doc["timestamp"] = GetTimestamp();
+  doc["pin"] = pin;
+  doc["value"] = value;
+  doc["si_unit"] = SensorTypeToString(read_type);
+}
+
 /**************************************************************************/
 /*!
     @brief  Logs a GPIO sensor event to the SD card.
@@ -644,20 +682,29 @@ const char *SensorTypeToString(wippersnapper_sensor_SensorType sensorType) {
 bool ws_sdcard::LogGPIOSensorEventToSD(
     uint8_t pin, float value, wippersnapper_sensor_SensorType read_type) {
   // Get the pin name in the correct format ("A0", "A1", etc.)
+  // TODO: Maybe send c_pin_name to sprintf and include A/D specifier from here?
   char c_pin_name[12];
   sprintf(c_pin_name, "A%d", pin);
 
-  // Get the RTC's timestamp
-  uint32_t timestamp = GetTimestamp();
-
   // Create the JSON document
   JsonDocument doc;
+  BuildJSONDoc(doc, c_pin_name, value, read_type);
 
-  doc["timestamp"] = timestamp;
-  doc["pin"] = c_pin_name;
-  doc["value"] = value;
-  doc["si_unit"] = SensorTypeToString(read_type);
-  serializeJson(doc, Serial);
+  // Serialize the JSON document
+#ifndef OFFLINE_MODE_DEBUG
+  // TODO: Make this an attempt to open and return false on failure
+  File32 file = _sd.open("log.json", FILE_WRITE);
+  BufferingPrint bufferedFile(file, 64); // Add buffering to the file
+  serializeJson(doc, file); // Serialize the JSON to the file in 64-byte chunks
+  // TODO: I am not sure if this works, consult PDF ch 4.7
+  bufferedFile.print("\n");
+  bufferedFile.flush(); // Send the remaining bytes
+#else
+  serializeJson(doc, Serial); // TODO: Add buffering here, too?
+  Serial.print("\n"); // JSON requires a newline at the end of each log line
+#endif
+
+  // TODO: Does this need to be a boolean?
   return true;
 }
 
@@ -679,17 +726,25 @@ bool ws_sdcard::LogGPIOSensorEventToSD(
   char c_pin_name[12];
   sprintf(c_pin_name, "A%d", pin);
 
-  // Get the RTC's timestamp
-  uint32_t timestamp = GetTimestamp();
-
-  // Append to the file in JSONL format
+  // Create the JSON document
   JsonDocument doc;
-  doc["timestamp"] = timestamp;
-  doc["pin"] = c_pin_name;
-  doc["value"] = value;
-  doc["si_unit"] = SensorTypeToString(read_type);
-  serializeJson(doc, Serial);
-  Serial.println(""); // JSON requires a newline at the end of each log line
+  BuildJSONDoc(doc, c_pin_name, value, read_type);
+
+  // Serialize the JSON document
+#ifndef OFFLINE_MODE_DEBUG
+  // TODO: Make this an attempt to open and return false on failure
+  File32 file = _sd.open("log.json", FILE_WRITE);
+  BufferingPrint bufferedFile(file, 64); // Add buffering to the file
+  serializeJson(doc, file); // Serialize the JSON to the file in 64-byte chunks
+  // TODO: I am not sure if this works, consult PDF ch 4.7
+  bufferedFile.print("\n");
+  bufferedFile.flush(); // Send the remaining bytes
+#else
+  serializeJson(doc, Serial); // TODO: Add buffering here, too?
+  Serial.print("\n"); // JSON requires a newline at the end of each log line
+#endif
+
+  // TODO: Does this need to be a boolean?
   return true;
 }
 
@@ -709,19 +764,30 @@ bool ws_sdcard::LogGPIOSensorEventToSD(
     uint8_t pin, bool value, wippersnapper_sensor_SensorType read_type) {
   // Get the pin name in the correct format ("A0", "A1", etc.)
   char c_pin_name[12];
-  sprintf(c_pin_name, "A%d", pin);
-
-  // Get the RTC's timestamp
-  uint32_t timestamp = GetTimestamp();
+  sprintf(c_pin_name, "D%d", pin);
 
   // Create the JSON document
   JsonDocument doc;
-  doc["timestamp"] = timestamp;
-  doc["pin"] = c_pin_name;
-  doc["value"] = value;
-  doc["si_unit"] = SensorTypeToString(read_type);
-  serializeJson(doc, Serial);
-  Serial.println("");
+  BuildJSONDoc(doc, c_pin_name, value, read_type);
+
+  // Serialize the JSON document
+#ifndef OFFLINE_MODE_DEBUG
+  // TODO: Make this an attempt to open and return false on failure
+  File32 file;
+  if (!_sd.open("log.json", FILE_WRITE)) {
+    WS_DEBUG_PRINTLN("[SD] FATAL Error - Unable to open JSON log file!");
+  }
+  BufferingPrint bufferedFile(file, 64); // Add buffering to the file
+  serializeJson(doc, file); // Serialize the JSON to the file in 64-byte chunks
+  // TODO: I am not sure if this works, consult PDF ch 4.7
+  bufferedFile.print("\n");
+  bufferedFile.flush(); // Send the remaining bytes
+#else
+  serializeJson(doc, Serial); // TODO: Add buffering here, too?
+  Serial.print("\n"); // JSON requires a newline at the end of each log line
+#endif
+
+  // TODO: Does this need to be a boolean?
   return true;
 }
 

src/provisioning/sdcard/ws_sdcard.cpp

@@ -16,6 +16,7 @@
 #define WS_SDCARD_H
 #include "RTClib.h"
 #include "SdFat.h"
+#include "StreamUtils.h"
 #include "Wippersnapper_V2.h"
 
 #define SD_FAT_TYPE 3
@@ -53,6 +54,12 @@ class ws_sdcard {
                        int num_sensors, const char *sensor_type_1,
                        const char *sensor_type_2);
 
+  void BuildJSONDoc(JsonDocument &doc, const char *pin, float value,
+                    wippersnapper_sensor_SensorType read_type);
+  void BuildJSONDoc(JsonDocument &doc, const char *pin, uint16_t value,
+                    wippersnapper_sensor_SensorType read_type);
+  void BuildJSONDoc(JsonDocument &doc, const char *pin, bool value,
+                    wippersnapper_sensor_SensorType read_type);
   bool LogGPIOSensorEventToSD(uint8_t pin, float value,
                               wippersnapper_sensor_SensorType read_type);
   bool LogGPIOSensorEventToSD(uint8_t pin, bool value,
@@ -78,6 +85,7 @@ class ws_sdcard {
   bool _use_test_data; ///< True if sample data is being used to test, instead
                        ///< of serial input, False otherwise.
   bool _wokwi_runner;  ///< True if `exportedBy` key is "wokwi", otherwise False
+  const char *_log_filename;           ///< Path to the log file
   RTC_DS3231 *_rtc_ds3231 = nullptr;   ///< DS3231 RTC object
   RTC_DS1307 *_rtc_ds1307 = nullptr;   ///< DS1307 RTC object
   RTC_PCF8523 *_rtc_pcf8523 = nullptr; ///< PCF8523 RTC object