Skip to content

Commit 806163f

Browse files
jasonsomersjasonsomers
authored
Merge branch 'wled:main' into main
2 parents bfe6fa1 + b7bfd6f commit 806163f

File tree

163 files changed

+7395
-6325
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

163 files changed

+7395
-6325
lines changed

‎.github/workflows/pr-merge.yaml‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,33 @@
11
name: Notify Discord on PR Merge
22
on:
3+
workflow_dispatch:
34
pull_request:
45
types: [closed]
56

67
jobs:
78
notify:
89
runs-on: ubuntu-latest
910
steps:
10-
- name: Send Discord notification
11-
shell: bash
11+
- name: Get User Permission
12+
id: checkAccess
13+
uses: actions-cool/check-user-permission@v2
14+
with:
15+
require: write
16+
username: ${{ github.triggering_actor }}
1217
env:
13-
DISCORD_WEBHOOK_BETA_TESTERS: ${{ secrets.DISCORD_WEBHOOK_BETA_TESTERS }}
14-
if: github.event.pull_request.merged == true
18+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
19+
- name: Check User Permission
20+
if: steps.checkAccess.outputs.require-result == 'false'
21+
run: |
22+
echo "${{ github.triggering_actor }} does not have permissions on this repo."
23+
echo "Current permission level is ${{ steps.checkAccess.outputs.user-permission }}"
24+
echo "Job originally triggered by ${{ github.actor }}"
25+
exit 1
26+
- name: Checkout code
27+
uses: actions/checkout@v3
28+
with:
29+
ref: ${{ github.event.pull_request.head.sha }} # This is dangerous without the first access check
30+
- name: Send Discord notification
31+
# if: github.event.pull_request.merged == true
1532
run: |
16-
curl -H "Content-Type: application/json" -d '{"content": "Pull Request #{{ github.event.pull_request.number }} merged by {{ github.actor }}"}' $DISCORD_WEBHOOK_BETA_TESTERS
33+
curl -H "Content-Type: application/json" -d '{"content": "Pull Request ${{ github.event.pull_request.number }} merged by ${{ github.actor }}"}' ${{ secrets.DISCORD_WEBHOOK_BETA_TESTERS }}

‎.github/workflows/usermods.yml‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
name: Usermod CI
2+
3+
on:
4+
push:
5+
paths:
6+
- usermods/**
7+
- .github/workflows/usermods.yml
8+
9+
jobs:
10+
11+
get_usermod_envs:
12+
name: Gather Usermods
13+
runs-on: ubuntu-latest
14+
steps:
15+
- uses: actions/checkout@v4
16+
- uses: actions/setup-python@v5
17+
with:
18+
python-version: '3.12'
19+
cache: 'pip'
20+
- name: Install PlatformIO
21+
run: pip install -r requirements.txt
22+
- name: Get default environments
23+
id: envs
24+
run: |
25+
echo "usermods=$(find usermods/ -name library.json | xargs dirname | xargs -n 1 basename | jq -R | grep -v PWM_fan | grep -v BME68X_v2| grep -v pixels_dice_tray | jq --slurp -c)" >> $GITHUB_OUTPUT
26+
outputs:
27+
usermods: ${{ steps.envs.outputs.usermods }}
28+
29+
30+
build:
31+
name: Build Enviornments
32+
runs-on: ubuntu-latest
33+
needs: get_usermod_envs
34+
strategy:
35+
fail-fast: false
36+
matrix:
37+
usermod: ${{ fromJSON(needs.get_usermod_envs.outputs.usermods) }}
38+
environment: [usermods_esp32, usermods_esp32c3, usermods_esp32s2, usermods_esp32s3]
39+
steps:
40+
- uses: actions/checkout@v4
41+
- name: Set up Node.js
42+
uses: actions/setup-node@v4
43+
with:
44+
node-version-file: '.nvmrc'
45+
cache: 'npm'
46+
- run: npm ci
47+
- name: Cache PlatformIO
48+
uses: actions/cache@v4
49+
with:
50+
path: |
51+
~/.platformio/.cache
52+
~/.buildcache
53+
build_output
54+
key: pio-${{ runner.os }}-${{ matrix.environment }}-${{ hashFiles('platformio.ini', 'pio-scripts/output_bins.py') }}-${{ hashFiles('wled00/**', 'usermods/**') }}
55+
restore-keys: pio-${{ runner.os }}-${{ matrix.environment }}-${{ hashFiles('platformio.ini', 'pio-scripts/output_bins.py') }}-
56+
- name: Set up Python
57+
uses: actions/setup-python@v5
58+
with:
59+
python-version: '3.12'
60+
cache: 'pip'
61+
- name: Install PlatformIO
62+
run: pip install -r requirements.txt
63+
- name: Add usermods environment
64+
run: |
65+
cp -v usermods/platformio_override.usermods.ini platformio_override.ini
66+
echo >> platformio_override.ini
67+
echo "custom_usermods = ${{ matrix.usermod }}" >> platformio_override.ini
68+
cat platformio_override.ini
69+
70+
- name: Build firmware
71+
run: pio run -e ${{ matrix.environment }}

‎package-lock.json‎

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pio-scripts/load_usermods.py‎

Lines changed: 44 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,14 @@
11
Import('env')
2-
import os.path
32
from collections import deque
43
from pathlib import Path # For OS-agnostic path manipulation
5-
from platformio.package.manager.library import LibraryPackageManager
4+
from click import secho
5+
from SCons.Script import Exit
6+
from platformio.builder.tools.piolib import LibBuilderBase
67

7-
usermod_dir = Path(env["PROJECT_DIR"]) / "usermods"
8-
all_usermods = [f for f in usermod_dir.iterdir() if f.is_dir() and f.joinpath('library.json').exists()]
8+
usermod_dir = Path(env["PROJECT_DIR"]).resolve() / "usermods"
99

10-
if env['PIOENV'] == "usermods":
11-
# Add all usermods
12-
env.GetProjectConfig().set(f"env:usermods", 'custom_usermods', " ".join([f.name for f in all_usermods]))
13-
14-
def find_usermod(mod: str):
10+
# Utility functions
11+
def find_usermod(mod: str) -> Path:
1512
"""Locate this library in the usermods folder.
1613
We do this to avoid needing to rename a bunch of folders;
1714
this could be removed later
@@ -22,51 +19,36 @@ def find_usermod(mod: str):
2219
return mp
2320
mp = usermod_dir / f"{mod}_v2"
2421
if mp.exists():
25-
return mp
22+
return mp
2623
mp = usermod_dir / f"usermod_v2_{mod}"
2724
if mp.exists():
2825
return mp
2926
raise RuntimeError(f"Couldn't locate module {mod} in usermods directory!")
3027

28+
def is_wled_module(dep: LibBuilderBase) -> bool:
29+
"""Returns true if the specified library is a wled module
30+
"""
31+
return usermod_dir in Path(dep.src_dir).parents or str(dep.name).startswith("wled-")
32+
33+
## Script starts here
34+
# Process usermod option
3135
usermods = env.GetProjectOption("custom_usermods","")
36+
37+
# Handle "all usermods" case
38+
if usermods == '*':
39+
usermods = [f.name for f in usermod_dir.iterdir() if f.is_dir() and f.joinpath('library.json').exists()]
40+
else:
41+
usermods = usermods.split()
42+
3243
if usermods:
3344
# Inject usermods in to project lib_deps
34-
proj = env.GetProjectConfig()
35-
deps = env.GetProjectOption('lib_deps')
36-
src_dir = proj.get("platformio", "src_dir")
37-
src_dir = src_dir.replace('\\','/')
38-
mod_paths = {mod: find_usermod(mod) for mod in usermods.split()}
39-
usermods = [f"{mod} = symlink://{path}" for mod, path in mod_paths.items()]
40-
proj.set("env:" + env['PIOENV'], 'lib_deps', deps + usermods)
41-
# Force usermods to be installed in to the environment build state before the LDF runs
42-
# Otherwise we won't be able to see them until it's too late to change their paths for LDF
43-
# Logic is largely borrowed from PlaformIO internals
44-
not_found_specs = []
45-
for spec in usermods:
46-
found = False
47-
for storage_dir in env.GetLibSourceDirs():
48-
#print(f"Checking {storage_dir} for {spec}")
49-
lm = LibraryPackageManager(storage_dir)
50-
if lm.get_package(spec):
51-
#print("Found!")
52-
found = True
53-
break
54-
if not found:
55-
#print("Missing!")
56-
not_found_specs.append(spec)
57-
if not_found_specs:
58-
lm = LibraryPackageManager(
59-
env.subst(os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV"))
60-
)
61-
for spec in not_found_specs:
62-
#print(f"LU: forcing install of {spec}")
63-
lm.install(spec)
64-
45+
symlinks = [f"symlink://{find_usermod(mod).resolve()}" for mod in usermods]
46+
env.GetProjectConfig().set("env:" + env['PIOENV'], 'lib_deps', env.GetProjectOption('lib_deps') + symlinks)
6547

6648
# Utility function for assembling usermod include paths
6749
def cached_add_includes(dep, dep_cache: set, includes: deque):
6850
""" Add dep's include paths to includes if it's not in the cache """
69-
if dep not in dep_cache:
51+
if dep not in dep_cache:
7052
dep_cache.add(dep)
7153
for include in dep.get_include_dirs():
7254
if include not in includes:
@@ -82,13 +64,6 @@ def cached_add_includes(dep, dep_cache: set, includes: deque):
8264

8365
# Our new wrapper
8466
def wrapped_ConfigureProjectLibBuilder(xenv):
85-
# Update usermod properties
86-
# Set libArchive before build actions are added
87-
for um in (um for um in xenv.GetLibBuilders() if usermod_dir in Path(um.src_dir).parents):
88-
build = um._manifest.get("build", {})
89-
build["libArchive"] = False
90-
um._manifest["build"] = build
91-
9267
# Call the wrapped function
9368
result = old_ConfigureProjectLibBuilder.clone(xenv)()
9469

@@ -102,12 +77,29 @@ def wrapped_ConfigureProjectLibBuilder(xenv):
10277
for dep in result.depbuilders:
10378
cached_add_includes(dep, processed_deps, extra_include_dirs)
10479

105-
for um in [dep for dep in result.depbuilders if usermod_dir in Path(dep.src_dir).parents]:
80+
wled_deps = [dep for dep in result.depbuilders if is_wled_module(dep)]
81+
82+
broken_usermods = []
83+
for dep in wled_deps:
10684
# Add the wled folder to the include path
107-
um.env.PrependUnique(CPPPATH=wled_dir)
85+
dep.env.PrependUnique(CPPPATH=str(wled_dir))
10886
# Add WLED's own dependencies
10987
for dir in extra_include_dirs:
110-
um.env.PrependUnique(CPPPATH=dir)
88+
dep.env.PrependUnique(CPPPATH=str(dir))
89+
# Enforce that libArchive is not set; we must link them directly to the executable
90+
if dep.lib_archive:
91+
broken_usermods.append(dep)
92+
93+
if broken_usermods:
94+
broken_usermods = [usermod.name for usermod in broken_usermods]
95+
secho(
96+
f"ERROR: libArchive=false is missing on usermod(s) {' '.join(broken_usermods)} -- modules will not compile in correctly",
97+
fg="red",
98+
err=True)
99+
Exit(1)
100+
101+
# Save the depbuilders list for later validation
102+
xenv.Replace(WLED_MODULES=wled_deps)
111103

112104
return result
113105

‎pio-scripts/validate_modules.py‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
import re
2+
from pathlib import Path # For OS-agnostic path manipulation
3+
from typing import Iterable
4+
from click import secho
5+
from SCons.Script import Action, Exit
6+
from platformio.builder.tools.piolib import LibBuilderBase
7+
8+
9+
def is_wled_module(env, dep: LibBuilderBase) -> bool:
10+
"""Returns true if the specified library is a wled module
11+
"""
12+
usermod_dir = Path(env["PROJECT_DIR"]).resolve() / "usermods"
13+
return usermod_dir in Path(dep.src_dir).parents or str(dep.name).startswith("wled-")
14+
15+
16+
def read_lines(p: Path):
17+
""" Read in the contents of a file for analysis """
18+
with p.open("r", encoding="utf-8", errors="ignore") as f:
19+
return f.readlines()
20+
21+
22+
def check_map_file_objects(map_file: list[str], dirs: Iterable[str]) -> set[str]:
23+
""" Identify which dirs contributed to the final build
24+
25+
Returns the (sub)set of dirs that are found in the output ELF
26+
"""
27+
# Pattern to match symbols in object directories
28+
# Join directories into alternation
29+
usermod_dir_regex = "|".join([re.escape(dir) for dir in dirs])
30+
# Matches nonzero address, any size, and any path in a matching directory
31+
object_path_regex = re.compile(r"0x0*[1-9a-f][0-9a-f]*\s+0x[0-9a-f]+\s+\S+[/\\](" + usermod_dir_regex + r")[/\\]\S+\.o")
32+
33+
found = set()
34+
for line in map_file:
35+
matches = object_path_regex.findall(line)
36+
for m in matches:
37+
found.add(m)
38+
return found
39+
40+
41+
def count_usermod_objects(map_file: list[str]) -> int:
42+
""" Returns the number of usermod objects in the usermod list """
43+
# Count the number of entries in the usermods table section
44+
return len([x for x in map_file if ".dtors.tbl.usermods.1" in x])
45+
46+
47+
def validate_map_file(source, target, env):
48+
""" Validate that all modules appear in the output build """
49+
build_dir = Path(env.subst("$BUILD_DIR"))
50+
map_file_path = build_dir / env.subst("${PROGNAME}.map")
51+
52+
if not map_file_path.exists():
53+
secho(f"ERROR: Map file not found: {map_file_path}", fg="red", err=True)
54+
Exit(1)
55+
56+
# Identify the WLED module builders, set by load_usermods.py
57+
module_lib_builders = env['WLED_MODULES']
58+
59+
# Extract the values we care about
60+
modules = {Path(builder.build_dir).name: builder.name for builder in module_lib_builders}
61+
secho(f"INFO: {len(modules)} libraries linked as WLED optional/user modules")
62+
63+
# Now parse the map file
64+
map_file_contents = read_lines(map_file_path)
65+
usermod_object_count = count_usermod_objects(map_file_contents)
66+
secho(f"INFO: {usermod_object_count} usermod object entries")
67+
68+
confirmed_modules = check_map_file_objects(map_file_contents, modules.keys())
69+
missing_modules = [modname for mdir, modname in modules.items() if mdir not in confirmed_modules]
70+
if missing_modules:
71+
secho(
72+
f"ERROR: No object files from {missing_modules} found in linked output!",
73+
fg="red",
74+
err=True)
75+
Exit(1)
76+
return None
77+
78+
Import("env")
79+
env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")])
80+
env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking linked optional modules (usermods) in map file'))

‎platformio.ini‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ extra_scripts =
116116
pre:pio-scripts/user_config_copy.py
117117
pre:pio-scripts/load_usermods.py
118118
pre:pio-scripts/build_ui.py
119+
post:pio-scripts/validate_modules.py ;; double-check the build output usermods
119120
; post:pio-scripts/obj-dump.py ;; convenience script to create a disassembly dump of the firmware (hardcore debugging)
120121

121122
# ------------------------------------------------------------------------------
@@ -659,5 +660,5 @@ build_flags = ${common.build_flags} ${esp32_idf_V4.build_flags} -D WLED_RELEASE_
659660
lib_deps = ${esp32_idf_V4.lib_deps}
660661
monitor_filters = esp32_exception_decoder
661662
board_build.flash_mode = dio
662-
; custom_usermods = *every folder with library.json* -- injected by pio-scripts/load_usermods.py
663+
custom_usermods = * ; Expands to all usermods in usermods folder
663664
board_build.partitions = ${esp32.extreme_partitions} ; We're gonna need a bigger boat

‎requirements.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ click==8.1.8
2020
# uvicorn
2121
colorama==0.4.6
2222
# via platformio
23-
h11==0.14.0
23+
h11==0.16.0
2424
# via
2525
# uvicorn
2626
# wsproto
@@ -38,7 +38,7 @@ pyelftools==0.32
3838
# via platformio
3939
pyserial==3.5
4040
# via platformio
41-
requests==2.32.3
41+
requests==2.32.4
4242
# via platformio
4343
semantic-version==2.10.0
4444
# via platformio
@@ -50,7 +50,7 @@ tabulate==0.9.0
5050
# via platformio
5151
typing-extensions==4.12.2
5252
# via anyio
53-
urllib3==2.3.0
53+
urllib3==2.5.0
5454
# via requests
5555
uvicorn==0.34.0
5656
# via platformio

0 commit comments

Comments
 (0)