Migration of persistent data from previous versions (#355)

* add migration and cleaning of configs; add unit tests

* keep client-state.jsn; opstore handled in migration

* update changelog

* fix UBSan error

* reduce Unit test log verbosity (faster execution)

Author: matth-x

CHANGELOG.md

@@ -21,6 +21,7 @@
 - Input validation for unsigned int Configs ([#344](https://github.com/matth-x/MicroOcpp/pull/344))
 - Support for TransactionMessageAttempts/-RetryInterval ([#345](https://github.com/matth-x/MicroOcpp/pull/345))
 - Heap profiler and custom allocator support ([#350](https://github.com/matth-x/MicroOcpp/pull/350))
+- Migration of persistent storage ([#355](https://github.com/matth-x/MicroOcpp/pull/355))
 
 ### Removed
 

CMakeLists.txt

@@ -180,7 +180,7 @@ target_compile_definitions(mo_unit_tests PUBLIC
     MO_PLATFORM=MO_PLATFORM_UNIX
     MO_NUMCONNECTORS=3
     MO_CUSTOM_TIMER
-    MO_DBG_LEVEL=MO_DL_DEBUG
+    MO_DBG_LEVEL=MO_DL_INFO
     MO_TRAFFIC_OUT
     MO_FILENAME_PREFIX="./mo_store/"
     MO_LocalAuthListMaxLength=8

src/MicroOcpp.cpp

@@ -259,21 +259,12 @@ void mocpp_initialize(Connection& connection, const char *bootNotificationCreden
     BootService::loadBootStats(filesystem, bootstats);
 
     if (autoRecover && bootstats.getBootFailureCount() > 3) {
-        MO_DBG_ERR("multiple initialization failures detected");
-        if (filesystem) {
-            bool success = FilesystemUtils::remove_if(filesystem, [] (const char *fname) -> bool {
-                return !strncmp(fname, "sd", strlen("sd")) ||
-                       !strncmp(fname, "tx", strlen("tx")) ||
-                       !strncmp(fname, "op", strlen("op")) ||
-                       !strncmp(fname, "sc-", strlen("sc-")) ||
-                       !strncmp(fname, "reservation", strlen("reservation"));
-            });
-            MO_DBG_ERR("clear local state files (recovery): %s", success ? "success" : "not completed");
-
-            bootstats = BootStats();
-        }
+        BootService::recover(filesystem, bootstats);
+        bootstats = BootStats();
     }
 
+    BootService::migrate(filesystem, bootstats);
+
     bootstats.bootNr++; //assign new boot number to this run
     BootService::storeBootStats(filesystem, bootstats);
 
@@ -373,13 +364,8 @@ void mocpp_initialize(Connection& connection, const char *bootNotificationCreden
     }
     credsJson.reset();
 
-    auto mocppVersion = declareConfiguration<const char*>("MicroOcppVersion", MO_VERSION, MO_KEYVALUE_FN, false, false, false);
-
     configuration_load();
 
-    if (mocppVersion) {
-        mocppVersion->setString(MO_VERSION);
-    }
     MO_DBG_INFO("initialized MicroOcpp v" MO_VERSION);
 }
 

src/MicroOcpp/Core/Configuration.cpp

@@ -239,4 +239,11 @@ bool configuration_save() {
     return success;
 }
 
+bool configuration_clean_unused() {
+    for (auto& container : configurationContainers) {
+        container->removeUnused();
+    }
+    return configuration_save();
+}
+
 } //end namespace MicroOcpp

src/MicroOcpp/Core/Configuration.h

@@ -36,5 +36,7 @@ bool configuration_load(const char *filename = nullptr);
 
 bool configuration_save();
 
+bool configuration_clean_unused(); //remove configs which haven't been accessed
+
 } //end namespace MicroOcpp
 #endif

src/MicroOcpp/Core/ConfigurationContainer.h

@@ -35,6 +35,8 @@ class ConfigurationContainer {
     virtual std::shared_ptr<Configuration> getConfiguration(const char *key) = 0;
 
     virtual void loadStaticKey(Configuration& config, const char *key) { } //possible optimization: can replace internal key with passed static key
+
+    virtual void removeUnused() { } //remove configs which haven't been accessed (optional and only if known)
 };
 
 class ConfigurationContainerVolatile : public ConfigurationContainer, public MemoryManaged {

src/MicroOcpp/Core/ConfigurationContainerFlash.cpp

@@ -334,6 +334,25 @@ class ConfigurationContainerFlash : public ConfigurationContainer, public Memory
         config.setKey(key);
         clearKeyPool(key);
     }
+
+    void removeUnused() override {
+        //if a config's key is still in the keyPool, we know it's unused because it has never been declared in FW (originates from an older FW version)
+
+        auto key = keyPool.begin();
+        while (key != keyPool.end()) {
+
+            for (auto config = configurations.begin(); config != configurations.end(); ++config) {
+                if ((*config)->getKey() == *key) {
+                    MO_DBG_DEBUG("remove unused config %s", (*config)->getKey());
+                    configurations.erase(config);
+                    break;
+                }
+            }
+
+            MO_FREE(*key);
+            key = keyPool.erase(key);
+        }
+    }
 };
 
 std::unique_ptr<ConfigurationContainer> makeConfigurationContainerFlash(std::shared_ptr<FilesystemAdapter> filesystem, const char *filename, bool accessible) {

src/MicroOcpp/Model/Boot/BootService.cpp

@@ -14,10 +14,6 @@
 #include <MicroOcpp/Platform.h>
 #include <MicroOcpp/Debug.h>
 
-#ifndef MO_BOOTSTATS_LONGTIME_MS
-#define MO_BOOTSTATS_LONGTIME_MS 180 * 1000
-#endif
-
 using namespace MicroOcpp;
 
 unsigned int PreBootQueue::getFrontRequestOpNr() {
@@ -71,6 +67,9 @@ void BootService::loop() {
     if (!executedLongTime && mocpp_tick_ms() - firstExecutionTimestamp >= MO_BOOTSTATS_LONGTIME_MS) {
         executedLongTime = true;
         MO_DBG_DEBUG("boot success timer reached");
+
+        configuration_clean_unused();
+
         BootStats bootstats;
         loadBootStats(filesystem, bootstats);
         bootstats.lastBootSuccess = bootstats.bootNr;
@@ -185,6 +184,14 @@ bool BootService::loadBootStats(std::shared_ptr<FilesystemAdapter> filesystem, B
             } else {
                 success = false;
             }
+
+            const char *microOcppVersionIn = (*json)["MicroOcppVersion"] | (const char*)nullptr;
+            if (microOcppVersionIn) {
+                auto ret = snprintf(bstats.microOcppVersion, sizeof(bstats.microOcppVersion), "%s", microOcppVersionIn);
+                if (ret < 0 || (size_t)ret >= sizeof(bstats.microOcppVersion)) {
+                    success = false;
+                }
+            } //else: version specifier can be missing after upgrade from pre 1.2.0 version
         } else {
             success = false;
         }
@@ -201,15 +208,58 @@ bool BootService::loadBootStats(std::shared_ptr<FilesystemAdapter> filesystem, B
     }
 }
 
-bool BootService::storeBootStats(std::shared_ptr<FilesystemAdapter> filesystem, BootStats bstats) {
+bool BootService::storeBootStats(std::shared_ptr<FilesystemAdapter> filesystem, BootStats& bstats) {
     if (!filesystem) {
         return false;
     }
 
-    auto json = initJsonDoc("v16.Boot.BootService", JSON_OBJECT_SIZE(2));
+    auto json = initJsonDoc("v16.Boot.BootService", JSON_OBJECT_SIZE(3));
 
     json["bootNr"] = bstats.bootNr;
     json["lastSuccess"] = bstats.lastBootSuccess;
+    json["MicroOcppVersion"] = (const char*)bstats.microOcppVersion;
 
     return FilesystemUtils::storeJson(filesystem, MO_FILENAME_PREFIX "bootstats.jsn", json);
 }
+
+bool BootService::recover(std::shared_ptr<FilesystemAdapter> filesystem, BootStats& bstats) {
+    if (!filesystem) {
+        return false;
+    }
+
+    bool success = FilesystemUtils::remove_if(filesystem, [] (const char *fname) -> bool {
+        return !strncmp(fname, "sd", strlen("sd")) ||
+                !strncmp(fname, "tx", strlen("tx")) ||
+                !strncmp(fname, "sc-", strlen("sc-")) ||
+                !strncmp(fname, "reservation", strlen("reservation")) ||
+                !strncmp(fname, "client-state", strlen("client-state"));
+    });
+    MO_DBG_ERR("clear local state files (recovery): %s", success ? "success" : "not completed");
+
+    return success;
+}
+
+bool BootService::migrate(std::shared_ptr<FilesystemAdapter> filesystem, BootStats& bstats) {
+    if (!filesystem) {
+        return false;
+    }
+
+    bool success = true;
+
+    if (strcmp(bstats.microOcppVersion, MO_VERSION)) {
+        MO_DBG_INFO("migrate persistent storage to MO v" MO_VERSION);
+        success = FilesystemUtils::remove_if(filesystem, [] (const char *fname) -> bool {
+            return !strncmp(fname, "sd", strlen("sd")) ||
+                    !strncmp(fname, "tx", strlen("tx")) ||
+                    !strncmp(fname, "op", strlen("op")) ||
+                    !strncmp(fname, "sc-", strlen("sc-")) ||
+                    !strcmp(fname, "client-state.cnf") ||
+                    !strcmp(fname, "arduino-ocpp.cnf") ||
+                    !strcmp(fname, "ocpp-creds.jsn");
+        });
+
+        snprintf(bstats.microOcppVersion, sizeof(bstats.microOcppVersion), "%s", MO_VERSION);
+        MO_DBG_DEBUG("clear local state files (migration): %s", success ? "success" : "not completed");
+    }
+    return success;
+}

src/MicroOcpp/Model/Boot/BootService.h

@@ -13,15 +13,23 @@
 
 #define MO_BOOT_INTERVAL_DEFAULT 60
 
+#ifndef MO_BOOTSTATS_LONGTIME_MS
+#define MO_BOOTSTATS_LONGTIME_MS 180 * 1000
+#endif
+
 namespace MicroOcpp {
 
+#define MO_BOOTSTATS_VERSION_SIZE 10
+
 struct BootStats {
     uint16_t bootNr = 0;
     uint16_t lastBootSuccess = 0;
 
     uint16_t getBootFailureCount() {
         return bootNr - lastBootSuccess;
     }
+
+    char microOcppVersion [MO_BOOTSTATS_VERSION_SIZE] = {'\0'};
 };
 
 enum class RegistrationStatus {
@@ -79,8 +87,12 @@ class BootService : public MemoryManaged {
     void notifyRegistrationStatus(RegistrationStatus status);
     void setRetryInterval(unsigned long interval);
 
-    static bool loadBootStats(std::shared_ptr<FilesystemAdapter> filesystem, BootStats& out);
-    static bool storeBootStats(std::shared_ptr<FilesystemAdapter> filesystem, BootStats bstats);
+    static bool loadBootStats(std::shared_ptr<FilesystemAdapter> filesystem, BootStats& bstats);
+    static bool storeBootStats(std::shared_ptr<FilesystemAdapter> filesystem, BootStats& bstats);
+
+    static bool recover(std::shared_ptr<FilesystemAdapter> filesystem, BootStats& bstats); //delete all persistent files which could lead to a crash
+
+    static bool migrate(std::shared_ptr<FilesystemAdapter> filesystem, BootStats& bstats); //migrate persistent storage if running on a new MO version
 };
 
 }

tests/Boot.cpp

@@ -12,6 +12,7 @@
 #include <MicroOcpp/Operations/BootNotification.h>
 #include <MicroOcpp/Operations/StatusNotification.h>
 #include <MicroOcpp/Operations/CustomOperation.h>
+#include <MicroOcpp/Model/Boot/BootService.h>
 #include <MicroOcpp/Model/Transactions/TransactionStore.h>
 #include <MicroOcpp/Debug.h>
 #include <catch2/catch.hpp>
@@ -309,5 +310,121 @@ TEST_CASE( "Boot Behavior" ) {
 
     }
 
+    SECTION("Auto recovery") {
+
+        //start transaction which will persist a few boot cycles, but then will be wiped by auto recovery
+        loop();
+        beginTransaction("mIdTag");
+        loop();
+        REQUIRE( getChargePointStatus() == ChargePointStatus_Charging );
+
+        declareConfiguration<const char*>("keepConfigOverRecovery", "originalVal");
+        configuration_save();
+
+        mocpp_deinitialize();
+
+        //MO has 2 unexpected power cycles. Probably just back luck - keep the local state and configuration
+
+        //Increase the power cycle counter manually because it's not possible to interrupt the MO lifecycle during unit tests
+        BootStats bootstats;
+        BootService::loadBootStats(filesystem, bootstats);
+        bootstats.bootNr += 2;
+        BootService::storeBootStats(filesystem, bootstats);
+
+        mocpp_initialize(loopback, ChargerCredentials(), filesystem, /*enable auto recovery*/ true);
+        BootService::loadBootStats(filesystem, bootstats);
+        REQUIRE( bootstats.getBootFailureCount() == 2 + 1 ); //two boot failures have been measured, +1 because each power cycle is counted as potentially failing until reaching the long runtime barrier
+
+        loop();
+
+        REQUIRE( getChargePointStatus() == ChargePointStatus_Charging );
+
+        REQUIRE( !strcmp(declareConfiguration<const char*>("keepConfigOverRecovery", "otherVal")->getString(), "originalVal") );
+
+        //check that the power cycle counter has been updated properly after the controller has been running stable over a long time
+        mtime += MO_BOOTSTATS_LONGTIME_MS;
+        loop();
+        BootService::loadBootStats(filesystem, bootstats);
+        REQUIRE( bootstats.getBootFailureCount() == 0 );
+
+        mocpp_deinitialize();
+
+        //MO has 10 power cycles without running for at least 3 minutes and wipes the local state, but keeps the configuration
+
+        BootStats bootstats2;
+        BootService::loadBootStats(filesystem, bootstats2);
+        bootstats2.bootNr += 10;
+        BootService::storeBootStats(filesystem, bootstats2);
+
+        mocpp_initialize(loopback, ChargerCredentials(), filesystem, /*enable auto recovery*/ true);
+
+        REQUIRE( !strcmp(declareConfiguration<const char*>("keepConfigOverRecovery", "otherVal")->getString(), "originalVal") );
+        BootStats bootstats3;
+        BootService::loadBootStats(filesystem, bootstats3);
+        REQUIRE( bootstats3.getBootFailureCount() == 0 + 1 ); //failure count is reset, but +1 because each power cycle is counted as potentially failing until reaching the long runtime barrier
+
+        loop();
+        REQUIRE( getChargePointStatus() == ChargePointStatus_Available );
+
+    }
+
+    SECTION("Migration") {
+
+        //migration removes files from previous MO versions which were running on the controller. This includes the
+        //transaction cache, but configs are preserved
+
+        auto old_opstore = filesystem->open(MO_FILENAME_PREFIX "opstore.jsn", "w"); //the opstore has been removed in MO v1.2.0
+        old_opstore->write("example content", sizeof("example content") - 1);
+        old_opstore.reset(); //flushes the file
+
+        loop();
+        auto tx = beginTransaction("mIdTag"); //tx store will also be removed
+        auto txNr = tx->getTxNr(); //remember this for later usage
+        tx.reset(); //reset this smart pointer
+        loop();
+        REQUIRE( getChargePointStatus() == ChargePointStatus_Charging );
+        endTransaction();
+        loop();
+
+        REQUIRE( getOcppContext()->getModel().getTransactionStore()->getTransaction(1, txNr) != nullptr ); //tx exists on flash
+
+        declareConfiguration<const char*>("keepConfigOverMigration", "originalVal"); //migration keeps configs
+        configuration_save();
+
+        mocpp_deinitialize();
+
+        //After a FW update, the tracked version number has changed
+        BootStats bootstats;
+        BootService::loadBootStats(filesystem, bootstats);
+        snprintf(bootstats.microOcppVersion, sizeof(bootstats.microOcppVersion), "oldFwVers");
+        BootService::storeBootStats(filesystem, bootstats);
+
+        mocpp_initialize(loopback, ChargerCredentials(), filesystem); //MO migrates here
+
+        size_t msize = 0;
+        REQUIRE( filesystem->stat(MO_FILENAME_PREFIX "opstore.jsn", &msize) != 0 ); //opstore has been removed
+
+        REQUIRE( getOcppContext()->getModel().getTransactionStore()->getTransaction(1, txNr) == nullptr ); //tx history entry has been removed
+
+        REQUIRE( !strcmp(declareConfiguration<const char*>("keepConfigOverMigration", "otherVal")->getString(), "originalVal") ); //config has been preserved
+    }
+
+    SECTION("Clean unused configs") {
+
+        declareConfiguration<const char*>("neverDeclaredInsideMO", "originalVal"); //unused configs will be cleared automatically after the controller has been running for a long time
+        configuration_save();
+
+        mocpp_deinitialize();
+
+        mocpp_initialize(loopback, ChargerCredentials(), filesystem); //all configs are loaded here, including the test config of this section
+        loop();
+
+        //unused configs will be cleared automatically after long time
+        mtime += MO_BOOTSTATS_LONGTIME_MS;
+        loop();
+
+        REQUIRE( !strcmp(declareConfiguration<const char*>("neverDeclaredInsideMO", "newVal")->getString(), "newVal") ); //config has been removed
+    }
+
     mocpp_deinitialize();
 }