Formalize dynarray system

Break out the dynamic array macros to a re-usable component and fix
the implementation.

Author: willmmiles

pio-scripts/dynarray.py

@@ -1,12 +1,19 @@
 # Add a section to the linker script to store our dynamic arrays
-# This is implemented as a pio post-script to ensure that our extra linker 
-# script fragments are processed last, after the base platform scripts have 
-# been loaded and all sections defined.
+# This is implemented as a pio post-script to ensure that we can
+# place our linker script at the correct point in the command arguments.
 Import("env")
+from pathlib import Path
 
-if env.get("PIOPLATFORM") == "espressif8266":
-    # Use a shim on this platform so we can share the same output blocks
-    # names as used by later platforms (ESP32)
-    env.Append(LINKFLAGS=["-Ttools/esp8266_rodata.ld"])
-
-env.Append(LINKFLAGS=["-Ttools/dynarray.ld"])
+platform = env.get("PIOPLATFORM")
+script_file = Path(f"tools/dynarray_{platform}.ld")
+if script_file.is_file():
+    linker_script = f"-T{script_file}"
+    if platform == "espressif32":
+        # For ESP32, the script must be added at the right point in the list
+        linkflags = env.get("LINKFLAGS", [])
+        idx = linkflags.index("memory.ld")    
+        linkflags.insert(idx+1, linker_script)    
+        env.Replace(LINKFLAGS=linkflags)
+    else:
+        # For other platforms, put it in last
+        env.Append(LINKFLAGS=[linker_script])

pio-scripts/validate_modules.py

@@ -4,7 +4,7 @@
 from click import secho
 from SCons.Script import Action, Exit
 from platformio.builder.tools.piolib import LibBuilderBase
-
+Import("env")
 
 def is_wled_module(env, dep: LibBuilderBase) -> bool:
   """Returns true if the specified library is a wled module
@@ -37,11 +37,13 @@ def check_map_file_objects(map_file: list[str], dirs: Iterable[str]) -> set[str]
             found.add(m)
     return found
 
+DYNARRAY_SECTION = ".dtors" if env.get("PIOPLATFORM") == "espressif8266" else ".dynarray"
+USERMODS_SECTION = f"{DYNARRAY_SECTION}.usermods.1"
 
 def count_usermod_objects(map_file: list[str]) -> int:
     """ Returns the number of usermod objects in the usermod list """
     # Count the number of entries in the usermods table section
-    return len([x for x in map_file if ".dynarray.usermods.1" in x])
+    return len([x for x in map_file if USERMODS_SECTION in x])
 
 
 def validate_map_file(source, target, env):
@@ -75,6 +77,5 @@ def validate_map_file(source, target, env):
         Exit(1)
     return None
 
-Import("env")
 env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")])
 env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking linked optional modules (usermods) in map file'))

tools/dynarray.ld

@@ -0,0 +1,10 @@
+/* ESP32 linker script fragment to add dynamic array section to binary */
+SECTIONS
+{
+  .dynarray :
+  {
+    . = ALIGN(0x10);
+    KEEP(*(SORT_BY_INIT_PRIORITY(.dynarray.*)))
+  } > default_rodata_seg
+}
+INSERT AFTER .flash.rodata;

tools/dynarray_espressif32.ld

@@ -0,0 +1,34 @@
+/* dynarray.h
+
+Macros for generating a "dynamic array", a static array of objects declared in different translation units
+
+*/
+
+#pragma once
+
+// Declare the beginning and ending elements of a dynamic array of 'type'.
+// This must be used in only one translation unit in your program for any given array.
+#define DECLARE_DYNARRAY(type, array_name) \
+  static type const DYNARRAY_BEGIN(array_name)[0] __attribute__((__section__(DYNARRAY_SECTION "." #array_name ".0"), unused)) = {}; \
+  static type const DYNARRAY_END(array_name)[0] __attribute__((__section__(DYNARRAY_SECTION "." #array_name ".99999"), unused)) = {};
+
+// Declare an object that is a member of a dynamic array.  "member name" must be unique; "array_section" is an integer for ordering items.
+// It is legal to define multiple items with the same section name; the order of those items will be up to the linker.
+#define DYNARRAY_MEMBER(type, array_name, member_name, array_section) type const member_name __attribute__((__section__(DYNARRAY_SECTION "." #array_name "." #array_section), used))
+
+#define DYNARRAY_BEGIN(array_name) array_name##_begin
+#define DYNARRAY_END(array_name) array_name##_end
+#define DYNARRAY_LENGTH(array_name) (&DYNARRAY_END(array_name)[0] - &DYNARRAY_BEGIN(array_name)[0])
+
+#ifdef ESP8266
+// ESP8266 linker script cannot be extended with a unique section for dynamic arrays.
+// We instead pack them in the ".dtors" section, as it's sorted and uploaded to the flash
+// (but will never be used in the embedded system)
+#define DYNARRAY_SECTION ".dtors"
+
+#else /* ESP8266 */
+
+// Use a unique named section; the linker script must be extended to ensure it's correctly placed.
+#define DYNARRAY_SECTION ".dynarray"
+
+#endif

tools/esp8266_rodata.ld

@@ -1,6 +1,7 @@
 #pragma once
 #ifndef WLED_FCN_DECLARE_H
 #define WLED_FCN_DECLARE_H
+#include <dynarray.h>
 
 /*
  * All globally accessible functions are declared here
@@ -366,7 +367,7 @@ namespace UsermodManager {
 };
 
 // Register usermods by building a static list via a linker section
-#define REGISTER_USERMOD(x) Usermod* const um_##x __attribute__((__section__(".dynarray.usermods.1"), used)) = &x
+#define REGISTER_USERMOD(x) DYNARRAY_MEMBER(Usermod*, usermods, um_##x, 1) = &x
 
 //usermod.cpp
 void userSetup();

wled00/dynarray.h

@@ -10,76 +10,80 @@
 // We stick them in the '.dtors' segment because it's always included by the linker scripts
 // even though it never gets called.  Who calls exit() in an embedded program anyways?
 // If someone ever does, though, it'll explode as these aren't function pointers.
-static Usermod * const _usermod_table_begin[0] __attribute__((__section__(".dynarray.usermods.0"), unused)) = {};
-static Usermod * const _usermod_table_end[0] __attribute__((__section__(".dynarray.usermods.99"), unused)) = {};
+DECLARE_DYNARRAY(Usermod*, usermods);
 
 static size_t getCount() {  
-  return &_usermod_table_end[0] - &_usermod_table_begin[0];
+  return DYNARRAY_LENGTH(usermods);
 }
 
 
 //Usermod Manager internals
-void UsermodManager::setup()             { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->setup(); }
-void UsermodManager::connected()         { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->connected(); }
-void UsermodManager::loop()              { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->loop();  }
-void UsermodManager::handleOverlayDraw() { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->handleOverlayDraw(); }
-void UsermodManager::appendConfigData(Print& dest)  { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->appendConfigData(dest); }
+void UsermodManager::setup()             { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->setup(); }
+void UsermodManager::connected()         { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->connected(); }
+void UsermodManager::loop()              { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->loop();  }
+void UsermodManager::handleOverlayDraw() { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->handleOverlayDraw(); }
+void UsermodManager::appendConfigData(Print& dest)  { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->appendConfigData(dest); }
 bool UsermodManager::handleButton(uint8_t b) {
   bool overrideIO = false;
-  for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) {
+  for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) {
     if ((*mod)->handleButton(b)) overrideIO = true;
   }
   return overrideIO;
 }
 bool UsermodManager::getUMData(um_data_t **data, uint8_t mod_id) {
-  for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) {
+  for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) {
     if (mod_id > 0 && (*mod)->getId() != mod_id) continue;  // only get data form requested usermod if provided
     if ((*mod)->getUMData(data)) return true;               // if usermod does provide data return immediately (only one usermod can provide data at one time)
   }
   return false;
 }
-void UsermodManager::addToJsonState(JsonObject& obj)    { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->addToJsonState(obj); }
+void UsermodManager::addToJsonState(JsonObject& obj)    { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->addToJsonState(obj); }
 void UsermodManager::addToJsonInfo(JsonObject& obj)     {
   auto um_id_list = obj.createNestedArray("um");  
-  for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) {
+  for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) {
     um_id_list.add((*mod)->getId());
     (*mod)->addToJsonInfo(obj);
   }
 }
-void UsermodManager::readFromJsonState(JsonObject& obj) { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->readFromJsonState(obj); }
-void UsermodManager::addToConfig(JsonObject& obj)       { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->addToConfig(obj); }
+void UsermodManager::readFromJsonState(JsonObject& obj) { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->readFromJsonState(obj); }
+void UsermodManager::addToConfig(JsonObject& obj)       { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->addToConfig(obj); }
 bool UsermodManager::readFromConfig(JsonObject& obj)    {
-  bool allComplete = true;
-  for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) {
+  Serial.printf_P(PSTR("Mods: %d  Begin: %08X  End: %08X\n"), getCount(), (intptr_t) &DYNARRAY_BEGIN(usermods)[0], (intptr_t) &DYNARRAY_END(usermods)[0]);
+  Usermod** volatile x =  (Usermod**) DYNARRAY_BEGIN(usermods);
+  Serial.printf_P(PSTR("X: %08X\n"), (intptr_t) x);
+  Serial.printf_P(PSTR("*X: %08X\n"), (intptr_t) *x);
+
+  bool allComplete = true;  
+  for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) {
     if (!(*mod)->readFromConfig(obj)) allComplete = false;
   }
   return allComplete;
 }
 #ifndef WLED_DISABLE_MQTT
-void UsermodManager::onMqttConnect(bool sessionPresent) { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->onMqttConnect(sessionPresent); }
+void UsermodManager::onMqttConnect(bool sessionPresent) { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->onMqttConnect(sessionPresent); }
 bool UsermodManager::onMqttMessage(char* topic, char* payload) {
-  for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) if ((*mod)->onMqttMessage(topic, payload)) return true;
+  for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) if ((*mod)->onMqttMessage(topic, payload)) return true;
   return false;
 }
 #endif
 #ifndef WLED_DISABLE_ESPNOW
 bool UsermodManager::onEspNowMessage(uint8_t* sender, uint8_t* payload, uint8_t len) {
-  for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) if ((*mod)->onEspNowMessage(sender, payload, len)) return true;
+  for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) if ((*mod)->onEspNowMessage(sender, payload, len)) return true;
   return false;
 }
 #endif
 bool UsermodManager::onUdpPacket(uint8_t* payload, size_t len) {
-  for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) if ((*mod)->onUdpPacket(payload, len)) return true;
+  for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) if ((*mod)->onUdpPacket(payload, len)) return true;
   return false;
 }
-void UsermodManager::onUpdateBegin(bool init) { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->onUpdateBegin(init); } // notify usermods that update is to begin
-void UsermodManager::onStateChange(uint8_t mode) { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->onStateChange(mode); } // notify usermods that WLED state changed
+void UsermodManager::onUpdateBegin(bool init) { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->onUpdateBegin(init); } // notify usermods that update is to begin
+void UsermodManager::onStateChange(uint8_t mode) { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->onStateChange(mode); } // notify usermods that WLED state changed
 
 /*
  * Enables usermods to lookup another Usermod.
  */
 Usermod* UsermodManager::lookup(uint16_t mod_id) {
-  for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) {
+  for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) {
     if ((*mod)->getId() == mod_id) {
       return *mod;
     }