[SD] Refactor checksum validation out, dont require it to operate

Author: brentru

data.json

@@ -0,0 +1 @@
+{"exportVersion": "1.0.0", "exportedBy": "wokwi", "exportedAt": "2024-10-28T18:58:23.976Z", "exportedFromDevice": {"board": "metroesp32s3", "firmwareVersion": "1.0.0-beta.93", "referenceVoltage": 2.6, "totalGPIOPins": 11, "totalAnalogPins": 6}, "components": [{"componentAPI": "digitalio", "name": "Button (D4)", "pinName": "D4", "type": "push_button", "mode": "DIGITAL", "sampleMode": "TIMER", "direction": "INPUT", "period": 5, "pull": "UP", "isPin": true}]}

generate_json_checksum.py

@@ -9,10 +9,11 @@
     print("Error: JSON file did not contain any data or path didn't match exactly, exiting...")
     exit(1)
 
-# convert json dict->string and sum
-str_json_data = json.dumps(json_data)
-sum_json_data = sum(bytearray(str_json_data.encode()))
-print('Sum (hex): ', hex(sum_json_data))
+if 'checksum' in json_data:
+    del json_data['checksum']
+
+# NOTE: This is important to re-serialize json data to match ArduinoJSON's compact serialization format
+str_json_data = json.dumps(json_data, separators=(',', ':'), sort_keys=True)
 
 # calculate checksum
 calculated_checksum = sum(bytearray(str_json_data.encode())) & 0xFF

src/provisioning/sdcard/ws_sdcard.cpp

@@ -441,6 +441,23 @@ bool ws_sdcard::CreateNewLogFile() {
   return true;
 }
 
+bool ws_sdcard::ValidateChecksum(JsonDocument &doc) {
+  int json_doc_checksum = doc["checksum"];
+  // Calculate the checksum of the JSON document without the checksum field
+  doc.remove("checksum");
+  String doc_without_checksum;
+  serializeJson(doc, doc_without_checksum);
+  uint8_t calculated_checksum = 0;
+  for (unsigned int i = 0; i < doc_without_checksum.length(); i++) {
+    calculated_checksum += doc_without_checksum[i];
+  }
+  calculated_checksum &= 0xFF; // take lsb
+
+  if (json_doc_checksum != calculated_checksum)
+    return false;
+  return true;
+}
+
 /**************************************************************************/
 /*!
     @brief  Searches for and parses the JSON configuration file and sets up
@@ -476,7 +493,6 @@ bool ws_sdcard::parseConfigFile() {
     error = deserializeJson(doc, json_test_data, max_json_len);
   }
 #endif
-
   // If the JSON document failed to deserialize - halt the running device and
   // print the error because it is not possible to continue running in offline
   // mode without a valid config file
@@ -485,11 +501,17 @@ bool ws_sdcard::parseConfigFile() {
                      String(error.c_str()));
     return false;
   }
-
   WS_DEBUG_PRINTLN("[SD] Successfully deserialized JSON config file!");
 
+  // Calculate the 8-bit checksum of the JSON file to ensure it is valid data
+  if (! ValidateChecksum(doc)) {
+    WS_DEBUG_PRINTLN("[SD] Checksum mismatch, file has been modified from its original state!");
+  }
+  WS_DEBUG_PRINTLN("[SD] JSON checksum OK!");
+
   // NOTE: This is only used by the CI runner, production builds do not run
   // this!
+  // TODO: Do we even need this anymore? We switched to a compile-time constant instead
   const char *exportedBy = doc["exportedBy"];
   if (strcmp(exportedBy, "wokwi") == 0) {
     _is_using_wokwi = true;

src/provisioning/sdcard/ws_sdcard.h

@@ -52,6 +52,7 @@ class ws_sdcard {
   bool LogDS18xSensorEventToSD(wippersnapper_ds18x20_Ds18x20Event *event_msg);
 
 private:
+  bool ValidateChecksum(JsonDocument &doc);
   bool ValidateJSON(const char *input);
   bool ValidateJSONKey(const char *key, const char *error_msg);
   void CheckIn(uint8_t max_digital_pins, uint8_t max_analog_pins,

tests/bin/offline/firmware.elf

@@ -0,0 +1,9 @@
+name: Offline Mode - Invalid Checksum Value
+version: 1
+author: Brent Rubell for Adafruit Industries
+
+steps:
+  - wait-serial: '[SD] Waiting for incoming JSON string...'
+  - write-serial: '{"exportVersion": "1.0.0", "exportedBy": "wokwi", "exportedAt": "2024-10-28T18:58:23.976Z", "exportedFromDevice": {"board": "metroesp32s3", "firmwareVersion": "1.0.0-beta.93", "referenceVoltage": 2.6, "totalGPIOPins": 11, "totalAnalogPins": 6}, "components": [{"componentAPI": "analogio", "name": "Analog Pin", "pinName": "D14", "type": "analog_pin", "mode": "ANALOG", "direction": "INPUT", "sampleMode": "TIMER", "analogReadMode": "PIN_VALUE", "period": 5, "isPin": true}], "checksum": 5}'
+  - write-serial: '\n'
+  - wait-serial: '[SD] Checksum mismatch, file has been modified from its original state!'

tests/scenarios/offline/test-invalid-checksum.scenario.yaml

@@ -4,7 +4,7 @@ author: Brent Rubell for Adafruit Industries
 
 steps:
   - wait-serial: '[SD] Waiting for incoming JSON string...'
-  - write-serial: '{"exportVersion":"1.0.0","exportedBy":"wokwi","exportedAt":"2024-10-28T18:58:23.976Z","exportedFromDevice":{"board":"metroesp32s3","firmwareVersion":"1.0.0-beta.93","referenceVoltage":2.6,"totalGPIOPins":11,"totalAnalogPins":6},"components":[{"componentAPI":"analogio","name":"Analog Pin","pinName":"D14","type":"analog_pin","mode":"ANALOG","direction":"INPUT","sampleMode":"TIMER","analogReadMode":"PIN_VALUE","period":5,"isPin":true}]}'
+  - write-serial: '{"exportVersion": "1.0.0", "exportedBy": "wokwi", "exportedAt": "2024-10-28T18:58:23.976Z", "exportedFromDevice": {"board": "metroesp32s3", "firmwareVersion": "1.0.0-beta.93", "referenceVoltage": 2.6, "totalGPIOPins": 11, "totalAnalogPins": 6}, "components": [{"componentAPI": "analogio", "name": "Analog Pin", "pinName": "D14", "type": "analog_pin", "mode": "ANALOG", "direction": "INPUT", "sampleMode": "TIMER", "analogReadMode": "PIN_VALUE", "period": 5, "isPin": true}], "checksum": 149}'
   - write-serial: '\n'
   - wait-serial: '[SD] Valid JSON string received!'
   - wait-serial: '[analogio] Added new pin:'

tests/scenarios/offline/test-log-analogin.scenario.yaml

@@ -4,7 +4,7 @@ author: Brent Rubell for Adafruit Industries
 
 steps:
   - wait-serial: '[SD] Waiting for incoming JSON string...'
-  - write-serial: '{"exportVersion":"1.0.0","exportedBy":"wokwi","exportedAt":"2024-10-28T18:58:23.976Z","exportedFromDevice":{"board":"metroesp32s3","firmwareVersion":"1.0.0-beta.93","referenceVoltage":2.6,"totalGPIOPins":11,"totalAnalogPins":6},"components":[{"componentAPI":"digitalio","name":"Button (D4)","pinName":"D4","type":"push_button","mode":"DIGITAL","sampleMode":"TIMER","direction":"INPUT","period":5,"pull":"UP","isPin":true}]}'
+  - write-serial: '{"checksum":183,"components":[{"componentAPI":"digitalio","direction":"INPUT","isPin":true,"mode":"DIGITAL","name":"Button (D4)","period":5,"pinName":"D4","pull":"UP","sampleMode":"TIMER","type":"push_button"}],"exportVersion":"1.0.0","exportedAt":"2024-10-28T18:58:23.976Z","exportedBy":"wokwi","exportedFromDevice":{"board":"metroesp32s3","firmwareVersion":"1.0.0-beta.93","referenceVoltage":2.6,"totalAnalogPins":6,"totalGPIOPins":11}}'
   - write-serial: '\n'
   - wait-serial: '[SD] Valid JSON string received!'
   - wait-serial: '[digitalio] Added new pin:'

tests/scenarios/offline/test-log-digital-in.scenario.yaml

@@ -4,7 +4,7 @@ author: Brent Rubell for Adafruit Industries
 
 steps:
   - wait-serial: '[SD] Waiting for incoming JSON string...'
-  - write-serial: '{"exportVersion":"1.0.0","exportedBy":"wokwi","exportedAt":"2024-10-28T18:58:23.976Z","exportedFromDevice":{"board":"metroesp32s3","firmwareVersion":"1.0.0-beta.93","referenceVoltage":2.6,"totalGPIOPins":11,"totalAnalogPins":6},"components":[{"componentAPI":"ds18x20","name":"DS18B20: Temperature Sensor (°F)","sensorTypeCount":2,"sensorType1":"object-temp-fahrenheit","sensorType2":"object-temp","pinName":"D6","sensorResolution":12,"period":5}]}'
+  - write-serial: '{"exportVersion": "1.0.0", "exportedBy": "wokwi", "exportedAt": "2024-10-28T18:58:23.976Z", "exportedFromDevice": {"board": "metroesp32s3", "firmwareVersion": "1.0.0-beta.93", "referenceVoltage": 2.6, "totalGPIOPins": 11, "totalAnalogPins": 6}, "components": [{"componentAPI": "ds18x20", "name": "DS18B20: Temperature Sensor", "sensorTypeCount": 2, "sensorType1": "object-temp-fahrenheit", "sensorType2": "object-temp", "pinName": "D6", "sensorResolution": 12, "period": 5}], "checksum": 34}'
   - write-serial: '\n'
   - wait-serial: '[SD] Valid JSON string received!'
   - wait-serial: '{"timestamp":0,"pin":"D6","value":0,"si_unit":"OBJECT_TEMPERATURE"}'

tests/scenarios/offline/test-log-ds18b20.scenario.yaml

@@ -0,0 +1,9 @@
+name: Offline Mode - Invalid Checksum Value
+version: 1
+author: Brent Rubell for Adafruit Industries
+
+steps:
+  - wait-serial: '[SD] Waiting for incoming JSON string...'
+  - write-serial: '{"exportVersion": "1.0.0", "exportedBy": "wokwi", "exportedAt": "2024-10-28T18:58:23.976Z", "exportedFromDevice": {"board": "metroesp32s3", "firmwareVersion": "1.0.0-beta.93", "referenceVoltage": 2.6, "totalGPIOPins": 11, "totalAnalogPins": 6}, "components": [{"componentAPI": "analogio", "name": "Analog Pin", "pinName": "D14", "type": "analog_pin", "mode": "ANALOG", "direction": "INPUT", "sampleMode": "TIMER", "analogReadMode": "PIN_VALUE", "period": 5, "isPin": true}], "checksum": 149}'
+  - write-serial: '\n'
+  - wait-serial: '[SD] JSON checksum OK!'