From b97b46ae12cdfc1b6c79ad827dc2fa50d96a949c Mon Sep 17 00:00:00 2001 From: Will Miles Date: Fri, 28 Nov 2025 09:23:49 -0500 Subject: [PATCH 1/7] Use new section for dynamic arrays Use a custom linker section to hold dynamic arrays such as the usermod list. This provides an extensible solution for wider use in the future (such as FX or Bus drivers), as well as IDF v5 compatibility. --- pio-scripts/dynarray.py | 12 ++++++++++++ pio-scripts/validate_modules.py | 2 +- platformio.ini | 1 + tools/dynarray.ld | 9 +++++++++ tools/esp8266_rodata.ld | 2 ++ wled00/fcn_declare.h | 2 +- wled00/um_manager.cpp | 4 ++-- 7 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 pio-scripts/dynarray.py create mode 100644 tools/dynarray.ld create mode 100644 tools/esp8266_rodata.ld diff --git a/pio-scripts/dynarray.py b/pio-scripts/dynarray.py new file mode 100644 index 0000000000..c8c1e6d2cc --- /dev/null +++ b/pio-scripts/dynarray.py @@ -0,0 +1,12 @@ +# Add a section to the linker script to store our dynamic arrays +# This is implemented as a pio post-script to ensure that our extra linker +# script fragments are processed last, after the base platform scripts have +# been loaded and all sections defined. +Import("env") + +if env.get("PIOPLATFORM") == "espressif8266": + # Use a shim on this platform so we can share the same output blocks + # names as used by later platforms (ESP32) + env.Append(LINKFLAGS=["-Ttools/esp8266_rodata.ld"]) + +env.Append(LINKFLAGS=["-Ttools/dynarray.ld"]) diff --git a/pio-scripts/validate_modules.py b/pio-scripts/validate_modules.py index d63b609ac8..b002af40be 100644 --- a/pio-scripts/validate_modules.py +++ b/pio-scripts/validate_modules.py @@ -41,7 +41,7 @@ def check_map_file_objects(map_file: list[str], dirs: Iterable[str]) -> set[str] def count_usermod_objects(map_file: list[str]) -> int: """ Returns the number of usermod objects in the usermod list """ # Count the number of entries in the usermods table section - return len([x for x in map_file if ".dtors.tbl.usermods.1" in x]) + return len([x for x in map_file if ".dynarray.usermods.1" in x]) def validate_map_file(source, target, env): diff --git a/platformio.ini b/platformio.ini index 60dedd473b..29949d33c0 100644 --- a/platformio.ini +++ b/platformio.ini @@ -135,6 +135,7 @@ extra_scripts = pre:pio-scripts/set_metadata.py post:pio-scripts/output_bins.py post:pio-scripts/strip-floats.py + post:pio-scripts/dynarray.py pre:pio-scripts/user_config_copy.py pre:pio-scripts/load_usermods.py pre:pio-scripts/build_ui.py diff --git a/tools/dynarray.ld b/tools/dynarray.ld new file mode 100644 index 0000000000..2c81217d35 --- /dev/null +++ b/tools/dynarray.ld @@ -0,0 +1,9 @@ +/* Linker script fragment to add dynamic array section to binary */ +SECTIONS +{ + .dynarray : + { + . = ALIGN(4); + KEEP(*(SORT_BY_INIT_PRIORITY(.dynarray.*))) + } > default_rodata_seg +} diff --git a/tools/esp8266_rodata.ld b/tools/esp8266_rodata.ld new file mode 100644 index 0000000000..0cb27df6d4 --- /dev/null +++ b/tools/esp8266_rodata.ld @@ -0,0 +1,2 @@ +/* Linker script fragment to shim ESP8266 section name to newer ESP32 standards */ +REGION_ALIAS("default_rodata_seg", iram1_0_seg) diff --git a/wled00/fcn_declare.h b/wled00/fcn_declare.h index 1c0b5a7a0d..e6a2112cd0 100644 --- a/wled00/fcn_declare.h +++ b/wled00/fcn_declare.h @@ -381,7 +381,7 @@ namespace UsermodManager { }; // Register usermods by building a static list via a linker section -#define REGISTER_USERMOD(x) Usermod* const um_##x __attribute__((__section__(".dtors.tbl.usermods.1"), used)) = &x +#define REGISTER_USERMOD(x) Usermod* const um_##x __attribute__((__section__(".dynarray.usermods.1"), used)) = &x //usermod.cpp void userSetup(); diff --git a/wled00/um_manager.cpp b/wled00/um_manager.cpp index 647757ad6f..2cefcdf347 100644 --- a/wled00/um_manager.cpp +++ b/wled00/um_manager.cpp @@ -10,8 +10,8 @@ // We stick them in the '.dtors' segment because it's always included by the linker scripts // even though it never gets called. Who calls exit() in an embedded program anyways? // If someone ever does, though, it'll explode as these aren't function pointers. -static Usermod * const _usermod_table_begin[0] __attribute__((__section__(".dtors.tbl.usermods.0"), unused)) = {}; -static Usermod * const _usermod_table_end[0] __attribute__((__section__(".dtors.tbl.usermods.99"), unused)) = {}; +static Usermod * const _usermod_table_begin[0] __attribute__((__section__(".dynarray.usermods.0"), unused)) = {}; +static Usermod * const _usermod_table_end[0] __attribute__((__section__(".dynarray.usermods.99"), unused)) = {}; static size_t getCount() { return &_usermod_table_end[0] - &_usermod_table_begin[0]; From 3bfbbab807cb9fb1c0755da4850d84130751005a Mon Sep 17 00:00:00 2001 From: Will Miles Date: Fri, 5 Dec 2025 14:25:27 -0500 Subject: [PATCH 2/7] Formalize dynarray system Break out the dynamic array macros to a re-usable component and fix the implementation. --- pio-scripts/dynarray.py | 25 +++++++++++------ pio-scripts/validate_modules.py | 7 +++-- tools/dynarray.ld | 9 ------ tools/dynarray_espressif32.ld | 10 +++++++ tools/esp8266_rodata.ld | 2 -- wled00/dynarray.h | 34 ++++++++++++++++++++++ wled00/fcn_declare.h | 3 +- wled00/um_manager.cpp | 50 ++++++++++++++++++--------------- 8 files changed, 93 insertions(+), 47 deletions(-) delete mode 100644 tools/dynarray.ld create mode 100644 tools/dynarray_espressif32.ld delete mode 100644 tools/esp8266_rodata.ld create mode 100644 wled00/dynarray.h diff --git a/pio-scripts/dynarray.py b/pio-scripts/dynarray.py index c8c1e6d2cc..2d3cfa90c5 100644 --- a/pio-scripts/dynarray.py +++ b/pio-scripts/dynarray.py @@ -1,12 +1,19 @@ # Add a section to the linker script to store our dynamic arrays -# This is implemented as a pio post-script to ensure that our extra linker -# script fragments are processed last, after the base platform scripts have -# been loaded and all sections defined. +# This is implemented as a pio post-script to ensure that we can +# place our linker script at the correct point in the command arguments. Import("env") +from pathlib import Path -if env.get("PIOPLATFORM") == "espressif8266": - # Use a shim on this platform so we can share the same output blocks - # names as used by later platforms (ESP32) - env.Append(LINKFLAGS=["-Ttools/esp8266_rodata.ld"]) - -env.Append(LINKFLAGS=["-Ttools/dynarray.ld"]) +platform = env.get("PIOPLATFORM") +script_file = Path(f"tools/dynarray_{platform}.ld") +if script_file.is_file(): + linker_script = f"-T{script_file}" + if platform == "espressif32": + # For ESP32, the script must be added at the right point in the list + linkflags = env.get("LINKFLAGS", []) + idx = linkflags.index("memory.ld") + linkflags.insert(idx+1, linker_script) + env.Replace(LINKFLAGS=linkflags) + else: + # For other platforms, put it in last + env.Append(LINKFLAGS=[linker_script]) diff --git a/pio-scripts/validate_modules.py b/pio-scripts/validate_modules.py index b002af40be..483a9324f7 100644 --- a/pio-scripts/validate_modules.py +++ b/pio-scripts/validate_modules.py @@ -4,7 +4,7 @@ from click import secho from SCons.Script import Action, Exit from platformio.builder.tools.piolib import LibBuilderBase - +Import("env") def is_wled_module(env, dep: LibBuilderBase) -> bool: """Returns true if the specified library is a wled module @@ -37,11 +37,13 @@ def check_map_file_objects(map_file: list[str], dirs: Iterable[str]) -> set[str] found.add(m) return found +DYNARRAY_SECTION = ".dtors" if env.get("PIOPLATFORM") == "espressif8266" else ".dynarray" +USERMODS_SECTION = f"{DYNARRAY_SECTION}.usermods.1" def count_usermod_objects(map_file: list[str]) -> int: """ Returns the number of usermod objects in the usermod list """ # Count the number of entries in the usermods table section - return len([x for x in map_file if ".dynarray.usermods.1" in x]) + return len([x for x in map_file if USERMODS_SECTION in x]) def validate_map_file(source, target, env): @@ -75,6 +77,5 @@ def validate_map_file(source, target, env): Exit(1) return None -Import("env") env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")]) env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking linked optional modules (usermods) in map file')) diff --git a/tools/dynarray.ld b/tools/dynarray.ld deleted file mode 100644 index 2c81217d35..0000000000 --- a/tools/dynarray.ld +++ /dev/null @@ -1,9 +0,0 @@ -/* Linker script fragment to add dynamic array section to binary */ -SECTIONS -{ - .dynarray : - { - . = ALIGN(4); - KEEP(*(SORT_BY_INIT_PRIORITY(.dynarray.*))) - } > default_rodata_seg -} diff --git a/tools/dynarray_espressif32.ld b/tools/dynarray_espressif32.ld new file mode 100644 index 0000000000..70ce51f19c --- /dev/null +++ b/tools/dynarray_espressif32.ld @@ -0,0 +1,10 @@ +/* ESP32 linker script fragment to add dynamic array section to binary */ +SECTIONS +{ + .dynarray : + { + . = ALIGN(0x10); + KEEP(*(SORT_BY_INIT_PRIORITY(.dynarray.*))) + } > default_rodata_seg +} +INSERT AFTER .flash.rodata; diff --git a/tools/esp8266_rodata.ld b/tools/esp8266_rodata.ld deleted file mode 100644 index 0cb27df6d4..0000000000 --- a/tools/esp8266_rodata.ld +++ /dev/null @@ -1,2 +0,0 @@ -/* Linker script fragment to shim ESP8266 section name to newer ESP32 standards */ -REGION_ALIAS("default_rodata_seg", iram1_0_seg) diff --git a/wled00/dynarray.h b/wled00/dynarray.h new file mode 100644 index 0000000000..f9e6de19d4 --- /dev/null +++ b/wled00/dynarray.h @@ -0,0 +1,34 @@ +/* dynarray.h + +Macros for generating a "dynamic array", a static array of objects declared in different translation units + +*/ + +#pragma once + +// Declare the beginning and ending elements of a dynamic array of 'type'. +// This must be used in only one translation unit in your program for any given array. +#define DECLARE_DYNARRAY(type, array_name) \ + static type const DYNARRAY_BEGIN(array_name)[0] __attribute__((__section__(DYNARRAY_SECTION "." #array_name ".0"), unused)) = {}; \ + static type const DYNARRAY_END(array_name)[0] __attribute__((__section__(DYNARRAY_SECTION "." #array_name ".99999"), unused)) = {}; + +// Declare an object that is a member of a dynamic array. "member name" must be unique; "array_section" is an integer for ordering items. +// It is legal to define multiple items with the same section name; the order of those items will be up to the linker. +#define DYNARRAY_MEMBER(type, array_name, member_name, array_section) type const member_name __attribute__((__section__(DYNARRAY_SECTION "." #array_name "." #array_section), used)) + +#define DYNARRAY_BEGIN(array_name) array_name##_begin +#define DYNARRAY_END(array_name) array_name##_end +#define DYNARRAY_LENGTH(array_name) (&DYNARRAY_END(array_name)[0] - &DYNARRAY_BEGIN(array_name)[0]) + +#ifdef ESP8266 +// ESP8266 linker script cannot be extended with a unique section for dynamic arrays. +// We instead pack them in the ".dtors" section, as it's sorted and uploaded to the flash +// (but will never be used in the embedded system) +#define DYNARRAY_SECTION ".dtors" + +#else /* ESP8266 */ + +// Use a unique named section; the linker script must be extended to ensure it's correctly placed. +#define DYNARRAY_SECTION ".dynarray" + +#endif diff --git a/wled00/fcn_declare.h b/wled00/fcn_declare.h index e6a2112cd0..67958314b4 100644 --- a/wled00/fcn_declare.h +++ b/wled00/fcn_declare.h @@ -1,6 +1,7 @@ #pragma once #ifndef WLED_FCN_DECLARE_H #define WLED_FCN_DECLARE_H +#include /* * All globally accessible functions are declared here @@ -381,7 +382,7 @@ namespace UsermodManager { }; // Register usermods by building a static list via a linker section -#define REGISTER_USERMOD(x) Usermod* const um_##x __attribute__((__section__(".dynarray.usermods.1"), used)) = &x +#define REGISTER_USERMOD(x) DYNARRAY_MEMBER(Usermod*, usermods, um_##x, 1) = &x //usermod.cpp void userSetup(); diff --git a/wled00/um_manager.cpp b/wled00/um_manager.cpp index 2cefcdf347..ef21303f0c 100644 --- a/wled00/um_manager.cpp +++ b/wled00/um_manager.cpp @@ -10,76 +10,80 @@ // We stick them in the '.dtors' segment because it's always included by the linker scripts // even though it never gets called. Who calls exit() in an embedded program anyways? // If someone ever does, though, it'll explode as these aren't function pointers. -static Usermod * const _usermod_table_begin[0] __attribute__((__section__(".dynarray.usermods.0"), unused)) = {}; -static Usermod * const _usermod_table_end[0] __attribute__((__section__(".dynarray.usermods.99"), unused)) = {}; +DECLARE_DYNARRAY(Usermod*, usermods); static size_t getCount() { - return &_usermod_table_end[0] - &_usermod_table_begin[0]; + return DYNARRAY_LENGTH(usermods); } //Usermod Manager internals -void UsermodManager::setup() { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->setup(); } -void UsermodManager::connected() { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->connected(); } -void UsermodManager::loop() { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->loop(); } -void UsermodManager::handleOverlayDraw() { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->handleOverlayDraw(); } -void UsermodManager::appendConfigData(Print& dest) { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->appendConfigData(dest); } +void UsermodManager::setup() { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->setup(); } +void UsermodManager::connected() { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->connected(); } +void UsermodManager::loop() { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->loop(); } +void UsermodManager::handleOverlayDraw() { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->handleOverlayDraw(); } +void UsermodManager::appendConfigData(Print& dest) { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->appendConfigData(dest); } bool UsermodManager::handleButton(uint8_t b) { bool overrideIO = false; - for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) { + for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) { if ((*mod)->handleButton(b)) overrideIO = true; } return overrideIO; } bool UsermodManager::getUMData(um_data_t **data, uint8_t mod_id) { - for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) { + for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) { if (mod_id > 0 && (*mod)->getId() != mod_id) continue; // only get data form requested usermod if provided if ((*mod)->getUMData(data)) return true; // if usermod does provide data return immediately (only one usermod can provide data at one time) } return false; } -void UsermodManager::addToJsonState(JsonObject& obj) { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->addToJsonState(obj); } +void UsermodManager::addToJsonState(JsonObject& obj) { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->addToJsonState(obj); } void UsermodManager::addToJsonInfo(JsonObject& obj) { auto um_id_list = obj.createNestedArray("um"); - for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) { + for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) { um_id_list.add((*mod)->getId()); (*mod)->addToJsonInfo(obj); } } -void UsermodManager::readFromJsonState(JsonObject& obj) { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->readFromJsonState(obj); } -void UsermodManager::addToConfig(JsonObject& obj) { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->addToConfig(obj); } +void UsermodManager::readFromJsonState(JsonObject& obj) { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->readFromJsonState(obj); } +void UsermodManager::addToConfig(JsonObject& obj) { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->addToConfig(obj); } bool UsermodManager::readFromConfig(JsonObject& obj) { - bool allComplete = true; - for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) { + Serial.printf_P(PSTR("Mods: %d Begin: %08X End: %08X\n"), getCount(), (intptr_t) &DYNARRAY_BEGIN(usermods)[0], (intptr_t) &DYNARRAY_END(usermods)[0]); + Usermod** volatile x = (Usermod**) DYNARRAY_BEGIN(usermods); + Serial.printf_P(PSTR("X: %08X\n"), (intptr_t) x); + Serial.printf_P(PSTR("*X: %08X\n"), (intptr_t) *x); + + bool allComplete = true; + for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) { if (!(*mod)->readFromConfig(obj)) allComplete = false; } return allComplete; } #ifndef WLED_DISABLE_MQTT -void UsermodManager::onMqttConnect(bool sessionPresent) { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->onMqttConnect(sessionPresent); } +void UsermodManager::onMqttConnect(bool sessionPresent) { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->onMqttConnect(sessionPresent); } bool UsermodManager::onMqttMessage(char* topic, char* payload) { - for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) if ((*mod)->onMqttMessage(topic, payload)) return true; + for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) if ((*mod)->onMqttMessage(topic, payload)) return true; return false; } #endif #ifndef WLED_DISABLE_ESPNOW bool UsermodManager::onEspNowMessage(uint8_t* sender, uint8_t* payload, uint8_t len) { - for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) if ((*mod)->onEspNowMessage(sender, payload, len)) return true; + for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) if ((*mod)->onEspNowMessage(sender, payload, len)) return true; return false; } #endif bool UsermodManager::onUdpPacket(uint8_t* payload, size_t len) { - for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) if ((*mod)->onUdpPacket(payload, len)) return true; + for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) if ((*mod)->onUdpPacket(payload, len)) return true; return false; } -void UsermodManager::onUpdateBegin(bool init) { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->onUpdateBegin(init); } // notify usermods that update is to begin -void UsermodManager::onStateChange(uint8_t mode) { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->onStateChange(mode); } // notify usermods that WLED state changed +void UsermodManager::onUpdateBegin(bool init) { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->onUpdateBegin(init); } // notify usermods that update is to begin +void UsermodManager::onStateChange(uint8_t mode) { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->onStateChange(mode); } // notify usermods that WLED state changed /* * Enables usermods to lookup another Usermod. */ Usermod* UsermodManager::lookup(uint16_t mod_id) { - for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) { + for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) { if ((*mod)->getId() == mod_id) { return *mod; } From ac1a4dfbfde4dc6199a6650c54dd4941914438ba Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sun, 1 Mar 2026 19:27:14 -0500 Subject: [PATCH 3/7] validate_modules: Remove obsolete code --- pio-scripts/validate_modules.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pio-scripts/validate_modules.py b/pio-scripts/validate_modules.py index 483a9324f7..7b6ff6cc84 100644 --- a/pio-scripts/validate_modules.py +++ b/pio-scripts/validate_modules.py @@ -3,15 +3,8 @@ from typing import Iterable from click import secho from SCons.Script import Action, Exit -from platformio.builder.tools.piolib import LibBuilderBase Import("env") -def is_wled_module(env, dep: LibBuilderBase) -> bool: - """Returns true if the specified library is a wled module - """ - usermod_dir = Path(env["PROJECT_DIR"]).resolve() / "usermods" - return usermod_dir in Path(dep.src_dir).parents or str(dep.name).startswith("wled-") - def read_lines(p: Path): """ Read in the contents of a file for analysis """ From 19292675d8a29175879530e994d1158ad37e2475 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sun, 1 Mar 2026 19:32:24 -0500 Subject: [PATCH 4/7] Support lib_deps syntax in custom_usermods Expand the parsing of custom_usermods to accept the same syntax as lib_deps. This allows external usermods to appear on the custom_usermods lines. Also includes comment handling. Co-Authored-By: Claude --- pio-scripts/load_usermods.py | 131 ++++++++++++++++++++++++++++++----- 1 file changed, 112 insertions(+), 19 deletions(-) diff --git a/pio-scripts/load_usermods.py b/pio-scripts/load_usermods.py index 38a08401e6..68b6d27a97 100644 --- a/pio-scripts/load_usermods.py +++ b/pio-scripts/load_usermods.py @@ -1,6 +1,8 @@ Import('env') from collections import deque from pathlib import Path # For OS-agnostic path manipulation +import re +from urllib.parse import urlparse from click import secho from SCons.Script import Exit from platformio.builder.tools.piolib import LibBuilderBase @@ -25,25 +27,115 @@ def find_usermod(mod: str) -> Path: return mp raise RuntimeError(f"Couldn't locate module {mod} in usermods directory!") -def is_wled_module(dep: LibBuilderBase) -> bool: - """Returns true if the specified library is a wled module +# Names of external/registry deps listed in custom_usermods. +# Populated during parsing below; read by is_wled_module() at configure time. +_custom_usermod_names: set[str] = set() + +# Matches any RFC-valid URL scheme (http, https, git, git+https, symlink, file, hg+ssh, etc.) +_URL_SCHEME_RE = re.compile(r'^[a-zA-Z][a-zA-Z0-9+.-]*://') +# SSH git URL: user@host:path (e.g. git@github.com:user/repo.git#tag) +_SSH_URL_RE = re.compile(r'^[^@\s]+@[^@:\s]+:[^:\s]') +# Explicit custom name: "LibName = " (PlatformIO [=] form) +_NAME_EQ_RE = re.compile(r'^([A-Za-z0-9_.-]+)\s*=\s*(\S.*)') + + +def _is_external_entry(line: str) -> bool: + """Return True if line is a lib_deps-style external/registry entry.""" + if _NAME_EQ_RE.match(line): # "LibName = " + return True + if _URL_SCHEME_RE.match(line): # https://, git://, symlink://, etc. + return True + if _SSH_URL_RE.match(line): # git@github.com:user/repo.git + return True + if '@' in line: # "owner/Name @ ^1.0.0" + return True + if re.match(r'^[^/\s]+/[^/\s]+$', line): # "owner/Name" + return True + return False + + +def _predict_dep_name(entry: str) -> str | None: + """Predict the library name PlatformIO will assign to this dep (best-effort). + + Accuracy relies on the library's manifest "name" matching the repo/package + name in the spec. This holds for well-authored libraries; the libArchive + check (which requires library.json) provides an early-failure safety net. """ - return usermod_dir in Path(dep.src_dir).parents or str(dep.name).startswith("wled-") - -## Script starts here -# Process usermod option -usermods = env.GetProjectOption("custom_usermods","") + entry = entry.strip() + # "LibName = " — name is given explicitly; always use it + m = _NAME_EQ_RE.match(entry) + if m: + return m.group(1).strip() + # URL scheme: extract name from path + if _URL_SCHEME_RE.match(entry): + parsed = urlparse(entry) + if parsed.netloc in ('github.com', 'gitlab.com', 'bitbucket.com'): + parts = [p for p in parsed.path.split('/') if p] + if len(parts) >= 2: + name = parts[1] + return name[:-4] if name.endswith('.git') else name + name = Path(parsed.path.rstrip('/')).name + return name.split('.')[0].strip() or None + # SSH git URL: git@github.com:user/repo.git#tag → repo + if _SSH_URL_RE.match(entry): + path_part = entry.split(':', 1)[1].split('#')[0].rstrip('/') + name = Path(path_part).name + return (name[:-4] if name.endswith('.git') else name) or None + # Versioned registry: "owner/Name @ version" → Name + if '@' in entry: + name_part = entry.split('@')[0].strip() + return name_part.split('/')[-1].strip() if '/' in name_part else name_part + # Plain registry: "owner/Name" → Name + if re.match(r'^[^/\s]+/[^/\s]+$', entry): + return entry.split('/')[-1].strip() + return None -# Handle "all usermods" case -if usermods == '*': - usermods = [f.name for f in usermod_dir.iterdir() if f.is_dir() and f.joinpath('library.json').exists()] -else: - usermods = usermods.split() -if usermods: - # Inject usermods in to project lib_deps - symlinks = [f"symlink://{find_usermod(mod).resolve()}" for mod in usermods] - env.GetProjectConfig().set("env:" + env['PIOENV'], 'lib_deps', env.GetProjectOption('lib_deps') + symlinks) +def is_wled_module(dep: LibBuilderBase) -> bool: + """Returns true if the specified library is a wled module.""" + return ( + usermod_dir in Path(dep.src_dir).parents + or str(dep.name).startswith("wled-") + or dep.name in _custom_usermod_names + ) + + +## Script starts here — parse custom_usermods +raw_usermods = env.GetProjectOption("custom_usermods", "") +usermods_libdeps: list[str] = [] + +for line in raw_usermods.splitlines(): + line = line.strip() + if not line or line.startswith('#') or line.startswith(';'): + continue + + if _is_external_entry(line): + # External URL or registry entry: pass through to lib_deps unchanged. + predicted = _predict_dep_name(line) + if predicted: + _custom_usermod_names.add(predicted) + else: + secho( + f"WARNING: Cannot determine library name for custom_usermods entry " + f"{line!r}. If it is not recognised as a WLED module at build time, " + f"ensure its library.json 'name' matches the repo name.", + fg="yellow", err=True) + usermods_libdeps.append(line) + else: + # Bare name(s): split on whitespace for backwards compatibility. + for token in line.split(): + if token == '*': + for mod_path in sorted(usermod_dir.iterdir()): + if mod_path.is_dir() and (mod_path / 'library.json').exists(): + _custom_usermod_names.add(mod_path.name) + usermods_libdeps.append(f"symlink://{mod_path.resolve()}") + else: + resolved = find_usermod(token) + _custom_usermod_names.add(resolved.name) + usermods_libdeps.append(f"symlink://{resolved.resolve()}") + +if usermods_libdeps: + env.GetProjectConfig().set("env:" + env['PIOENV'], 'lib_deps', env.GetProjectOption('lib_deps') + usermods_libdeps) # Utility function for assembling usermod include paths def cached_add_includes(dep, dep_cache: set, includes: deque): @@ -93,9 +185,10 @@ def wrapped_ConfigureProjectLibBuilder(xenv): if broken_usermods: broken_usermods = [usermod.name for usermod in broken_usermods] secho( - f"ERROR: libArchive=false is missing on usermod(s) {' '.join(broken_usermods)} -- modules will not compile in correctly", - fg="red", - err=True) + f"ERROR: libArchive=false is missing on usermod(s) {' '.join(broken_usermods)} -- " + f"modules will not compile in correctly. Add '\"build\": {{\"libArchive\": false}}' " + f"to their library.json.", + fg="red", err=True) Exit(1) # Save the depbuilders list for later validation From 210b4d8f54f50b56ef26581efc5641a4dd494d0b Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sun, 1 Mar 2026 21:30:21 -0500 Subject: [PATCH 5/7] Add external usermod example --- platformio_override.sample.ini | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/platformio_override.sample.ini b/platformio_override.sample.ini index 2fc9aacfb4..e840eb4334 100644 --- a/platformio_override.sample.ini +++ b/platformio_override.sample.ini @@ -538,9 +538,21 @@ monitor_filters = esp32_exception_decoder # 433MHz RF remote example for esp32dev [env:esp32dev_usermod_RF433] extends = env:esp32dev -build_flags = ${env:esp32dev.build_flags} -D USERMOD_RF433 -lib_deps = ${env:esp32dev.lib_deps} - sui77/rc-switch @ 2.6.4 +custom_usermods = + ${env:esp32dev.custom_usermods} + RF433 + +# External usermod from a git repository. +# The library's `library.json` must include `"build": {"libArchive": false}`. +# The name PlatformIO assigns is taken from the library's `library.json` "name" field. +# If that name doesn't match the repo name in the URL, use the "LibName = URL" form +# shown in the commented-out line below to supply the name explicitly. +[env:esp32dev_external_usermod] +extends = env:esp32dev +custom_usermods = + ${env:esp32dev.custom_usermods} + https://github.com/wled/wled-usermod-example.git#main + # ------------------------------------------------------------------------------ # Hub75 examples From 05498f2ae4ef18dc4d14c9a5641b6c9237b469e5 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Fri, 6 Mar 2026 23:21:04 -0500 Subject: [PATCH 6/7] Apply fixes from code review h/t @coderabbitai --- pio-scripts/load_usermods.py | 8 +++++--- wled00/um_manager.cpp | 5 ----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/pio-scripts/load_usermods.py b/pio-scripts/load_usermods.py index 68b6d27a97..a3aa779c9d 100644 --- a/pio-scripts/load_usermods.py +++ b/pio-scripts/load_usermods.py @@ -73,9 +73,11 @@ def _predict_dep_name(entry: str) -> str | None: parts = [p for p in parsed.path.split('/') if p] if len(parts) >= 2: name = parts[1] - return name[:-4] if name.endswith('.git') else name - name = Path(parsed.path.rstrip('/')).name - return name.split('.')[0].strip() or None + else: + name = Path(parsed.path.rstrip('/')).name.strip() + if name.endswith('.git'): + name = name[:-4] + return name or None # SSH git URL: git@github.com:user/repo.git#tag → repo if _SSH_URL_RE.match(entry): path_part = entry.split(':', 1)[1].split('#')[0].rstrip('/') diff --git a/wled00/um_manager.cpp b/wled00/um_manager.cpp index ef21303f0c..504b5ba97c 100644 --- a/wled00/um_manager.cpp +++ b/wled00/um_manager.cpp @@ -48,11 +48,6 @@ void UsermodManager::addToJsonInfo(JsonObject& obj) { void UsermodManager::readFromJsonState(JsonObject& obj) { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->readFromJsonState(obj); } void UsermodManager::addToConfig(JsonObject& obj) { for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) (*mod)->addToConfig(obj); } bool UsermodManager::readFromConfig(JsonObject& obj) { - Serial.printf_P(PSTR("Mods: %d Begin: %08X End: %08X\n"), getCount(), (intptr_t) &DYNARRAY_BEGIN(usermods)[0], (intptr_t) &DYNARRAY_END(usermods)[0]); - Usermod** volatile x = (Usermod**) DYNARRAY_BEGIN(usermods); - Serial.printf_P(PSTR("X: %08X\n"), (intptr_t) x); - Serial.printf_P(PSTR("*X: %08X\n"), (intptr_t) *x); - bool allComplete = true; for (auto mod = DYNARRAY_BEGIN(usermods); mod < DYNARRAY_END(usermods); ++mod) { if (!(*mod)->readFromConfig(obj)) allComplete = false; From 7f44396f7f320f52fb937de1df25f9aa989029e1 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sat, 7 Mar 2026 08:55:42 -0500 Subject: [PATCH 7/7] validate_modules: Support LTO When LTO is enabled, the map file no longer provides a positive indication of whether a given object file has contributed to the final binary. Instead use nm to parse the debug data in the .elf file. Co-Authored-By: Claude --- pio-scripts/load_usermods.py | 8 ++++ pio-scripts/validate_modules.py | 73 ++++++++++++++++++++++++++------- 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/pio-scripts/load_usermods.py b/pio-scripts/load_usermods.py index a3aa779c9d..18852ff30b 100644 --- a/pio-scripts/load_usermods.py +++ b/pio-scripts/load_usermods.py @@ -180,6 +180,14 @@ def wrapped_ConfigureProjectLibBuilder(xenv): # Add WLED's own dependencies for dir in extra_include_dirs: dep.env.PrependUnique(CPPPATH=str(dir)) + # Ensure debug info is emitted for this module's source files. + # validate_modules.py uses `nm --defined-only -l` on the final ELF to check + # that each module has at least one symbol placed in the binary. The -l flag + # reads DWARF debug sections to map placed symbols back to their original source + # files; without -g those sections are absent and the check cannot attribute any + # symbol to a specific module. We scope this to usermods only — the main WLED + # build and other libraries are unaffected. + dep.env.AppendUnique(CCFLAGS=["-g"]) # Enforce that libArchive is not set; we must link them directly to the executable if dep.lib_archive: broken_usermods.append(dep) diff --git a/pio-scripts/validate_modules.py b/pio-scripts/validate_modules.py index 7b6ff6cc84..ae02e1f80d 100644 --- a/pio-scripts/validate_modules.py +++ b/pio-scripts/validate_modules.py @@ -1,6 +1,7 @@ +import os import re +import subprocess from pathlib import Path # For OS-agnostic path manipulation -from typing import Iterable from click import secho from SCons.Script import Action, Exit Import("env") @@ -12,24 +13,64 @@ def read_lines(p: Path): return f.readlines() -def check_map_file_objects(map_file: list[str], dirs: Iterable[str]) -> set[str]: - """ Identify which dirs contributed to the final build +def _get_nm_path(env) -> str: + """ Derive the nm tool path from the build environment """ + if "NM" in env: + return env.subst("$NM") + # Derive from the C compiler: xtensa-esp32-elf-gcc → xtensa-esp32-elf-nm + cc = env.subst("$CC") + nm = re.sub(r'(gcc|g\+\+)$', 'nm', os.path.basename(cc)) + return os.path.join(os.path.dirname(cc), nm) - Returns the (sub)set of dirs that are found in the output ELF + +def check_elf_modules(elf_path: Path, env, module_lib_builders) -> set[str]: + """ Check which modules have at least one defined symbol placed in the ELF. + + The map file is not a reliable source for this: with LTO, original object + file paths are replaced by temporary ltrans.o partitions in all output + sections, making per-module attribution impossible from the map alone. + Instead we invoke nm --defined-only -l on the ELF, which uses DWARF debug + info to attribute each placed symbol to its original source file. + + Requires usermod libraries to be compiled with -g so that DWARF sections + are present in the ELF. load_usermods.py injects -g for all WLED modules + via dep.env.AppendUnique(CCFLAGS=["-g"]). + + Returns the set of build_dir basenames for confirmed modules. """ - # Pattern to match symbols in object directories - # Join directories into alternation - usermod_dir_regex = "|".join([re.escape(dir) for dir in dirs]) - # Matches nonzero address, any size, and any path in a matching directory - object_path_regex = re.compile(r"0x0*[1-9a-f][0-9a-f]*\s+0x[0-9a-f]+\s+\S+[/\\](" + usermod_dir_regex + r")[/\\]\S+\.o") + nm_path = _get_nm_path(env) + try: + result = subprocess.run( + [nm_path, "--defined-only", "-l", str(elf_path)], + capture_output=True, text=True, errors="ignore", timeout=120, + ) + nm_output = result.stdout + except (subprocess.TimeoutExpired, FileNotFoundError, OSError) as e: + secho(f"WARNING: nm failed ({e}); skipping per-module validation", fg="yellow", err=True) + return {Path(b.build_dir).name for b in module_lib_builders} # conservative pass + + # Build a filtered set of lines that have a nonzero address. + # nm --defined-only still includes debugging symbols (type 'N') such as the + # per-CU markers GCC emits in .debug_info (e.g. "usermod_example_cpp_6734d48d"). + # These live at address 0x00000000 in their debug section — not in any load + # segment — so filtering them out leaves only genuinely placed symbols. + placed_lines = [ + line for line in nm_output.splitlines() + if (parts := line.split(None, 1)) and parts[0].lstrip('0') + ] + placed_output = "\n".join(placed_lines) found = set() - for line in map_file: - matches = object_path_regex.findall(line) - for m in matches: - found.add(m) + for builder in module_lib_builders: + # builder.src_dir is the library source directory (used by is_wled_module() too) + src_dir = str(builder.src_dir).rstrip("/\\") + # Guard against prefix collisions (e.g. /path/to/mod vs /path/to/mod-extra) + # by requiring a path separator immediately after the directory name. + if re.search(re.escape(src_dir) + r'[/\\]', placed_output): + found.add(Path(builder.build_dir).name) return found + DYNARRAY_SECTION = ".dtors" if env.get("PIOPLATFORM") == "espressif8266" else ".dynarray" USERMODS_SECTION = f"{DYNARRAY_SECTION}.usermods.1" @@ -60,11 +101,13 @@ def validate_map_file(source, target, env): usermod_object_count = count_usermod_objects(map_file_contents) secho(f"INFO: {usermod_object_count} usermod object entries") - confirmed_modules = check_map_file_objects(map_file_contents, modules.keys()) + elf_path = build_dir / env.subst("${PROGNAME}.elf") + confirmed_modules = check_elf_modules(elf_path, env, module_lib_builders) + missing_modules = [modname for mdir, modname in modules.items() if mdir not in confirmed_modules] if missing_modules: secho( - f"ERROR: No object files from {missing_modules} found in linked output!", + f"ERROR: No symbols from {missing_modules} found in linked output!", fg="red", err=True) Exit(1)