From 31b19ddf6692f1aaabd28280397d9db984f9625a Mon Sep 17 00:00:00 2001 From: Jamie Smith Date: Sun, 10 Aug 2025 09:09:39 -0700 Subject: [PATCH 01/15] Start importing PlatformIO target adapter code --- tools/cmake/mbed_create_distro.cmake | 2 +- tools/pyproject.toml | 5 + tools/python/mbed_platformio/__init__.py | 0 tools/python/mbed_platformio/build-mbed-ce.py | 306 ++++++++++++++++++ .../cmake_to_scons_converter.py | 119 +++++++ tools/python/mbed_platformio/pio_variants.py | 21 ++ 6 files changed, 452 insertions(+), 1 deletion(-) create mode 100644 tools/python/mbed_platformio/__init__.py create mode 100644 tools/python/mbed_platformio/build-mbed-ce.py create mode 100644 tools/python/mbed_platformio/cmake_to_scons_converter.py create mode 100644 tools/python/mbed_platformio/pio_variants.py diff --git a/tools/cmake/mbed_create_distro.cmake b/tools/cmake/mbed_create_distro.cmake index 6dc83e4d174..60598499121 100644 --- a/tools/cmake/mbed_create_distro.cmake +++ b/tools/cmake/mbed_create_distro.cmake @@ -142,7 +142,7 @@ function(mbed_extract_flags NAME) # ARGN: modules... foreach(SUBMODULE ${SUBMODULES}) if(NOT "${SUBMODULE}" MATCHES "::@") # remove CMake internal CMAKE_DIRECTORY_ID_SEP markers # Remove LINK_ONLY genexes from target_link_libraries(... PRIVATE). We can ignore things wrapped in these - # because they will already have been handled by the target_link_libraries earlier on. + # because they are for private dependencies. if(NOT "${SUBMODULE}" MATCHES "\\$") if(NOT ${SUBMODULE} IN_LIST COMPLETED_MODULES) list(APPEND REMAINING_MODULES ${SUBMODULE}) diff --git a/tools/pyproject.toml b/tools/pyproject.toml index 0087c13fcb0..4e06336e688 100644 --- a/tools/pyproject.toml +++ b/tools/pyproject.toml @@ -72,6 +72,11 @@ unit-tests = [ "lxml" ] +platformio = [ + "SCons", + "platformio" +] + [tool.hatch.build.targets.wheel] packages = [ "python/mbed_host_tests", diff --git a/tools/python/mbed_platformio/__init__.py b/tools/python/mbed_platformio/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tools/python/mbed_platformio/build-mbed-ce.py b/tools/python/mbed_platformio/build-mbed-ce.py new file mode 100644 index 00000000000..8b3c6f5fda0 --- /dev/null +++ b/tools/python/mbed_platformio/build-mbed-ce.py @@ -0,0 +1,306 @@ +""" +PlatformIO build file for Mbed OS Community Edition. + +This script acts as an SCons buildfile which gets configuration from PlatformIO, configures Mbed (if needed), +and returns information about the configuration to the PIO build system. +""" +from __future__ import annotations + +import pathlib +from pathlib import Path +import json +import sys +import os + +from SCons.Script import DefaultEnvironment, ARGUMENTS +from SCons.Environment import Base as Environment +from platformio.proc import exec_command +import click + +from .pio_variants import PIO_VARIANT_TO_MBED_TARGET +from .cmake_to_scons_converter import build_library + +env: Environment = DefaultEnvironment() +platform = env.PioPlatform() +board = env.BoardConfig() + +# Directories +FRAMEWORK_DIR = platform.get_package_dir("framework-mbed-ce") +BUILD_DIR = Path(env.subst("$BUILD_DIR")) +PROJECT_DIR = Path(env.subst("$PROJECT_DIR")) +PROJECT_SRC_DIR = Path(env.subst("$PROJECT_SRC_DIR")) +CMAKE_API_DIR = BUILD_DIR / ".cmake" / "api" / "v1" +CMAKE_API_QUERY_DIR = CMAKE_API_DIR / "query" +CMAKE_API_REPLY_DIR = CMAKE_API_DIR / "reply" + +PROJECT_CMAKELISTS_TXT = PROJECT_DIR / "CMakeLists.txt" +PROJECT_MBED_APP_JSON5 = PROJECT_DIR / "mbed_app.json5" + +def get_mbed_target(): + board_type = env.subst("$BOARD") + variant = ( + PIO_VARIANT_TO_MBED_TARGET[board_type] + if board_type in PIO_VARIANT_TO_MBED_TARGET + else board_type.upper() + ) + return board.get("build.mbed_variant", variant) + +def is_proper_mbed_ce_project(): + return all( + path.is_file() + for path in ( + PROJECT_MBED_APP_JSON5, + PROJECT_CMAKELISTS_TXT + ) + ) + +def create_default_project_files(): + if not PROJECT_CMAKELISTS_TXT.exists(): + PROJECT_CMAKELISTS_TXT.write_text( +""" # Default CMakeLists.txt for Mbed CE, created by PlatformIO +cmake_minimum_required(VERSION 3.19) +cmake_policy(VERSION 3.19...3.22) + +set(MBED_APP_JSON_PATH mbed_app.json5) + +include(mbed-os/tools/cmake/mbed_toolchain_setup.cmake) +project(PlatformIOMbedProject + LANGUAGES C CXX ASM) +include(mbed_project_setup) + +add_subdirectory(mbed-os) +""") + + if not PROJECT_MBED_APP_JSON5.exists(): + PROJECT_MBED_APP_JSON5.write_text( +""" +{ + "target_overrides": { + "*": { + "platform.stdio-baud-rate": 115200, + "platform.stdio-buffered-serial": 1, + + // Uncomment to use mbed-baremetal instead of mbed-os + // "target.application-profile": "bare-metal" + } + } +} +""" + ) + +def is_cmake_reconfigure_required(): + cmake_cache_file = BUILD_DIR / "CMakeCache.txt" + cmake_config_files = [ + PROJECT_MBED_APP_JSON5, + PROJECT_CMAKELISTS_TXT + ] + ninja_buildfile = BUILD_DIR / "build.ninja" + + if not CMAKE_API_REPLY_DIR.is_dir() or not not any(CMAKE_API_REPLY_DIR.iterdir()): + return True + if not cmake_cache_file.exists(): + return True + if not ninja_buildfile.exists(): + return True + + cache_file_mtime = cmake_cache_file.stat().st_mtime + if any( + f.stat().st_mtime > cache_file_mtime + for f in cmake_config_files + ): + return True + + return False + + +def run_cmake(src_dir: pathlib.Path, build_dir: pathlib.Path, extra_args=None): + cmd = [ + str(pathlib.Path(platform.get_package_dir("tool-cmake")) / "bin" / "cmake"), + "-S", + src_dir, + "-B", + build_dir, + "-G", + "Ninja", + ] + + if extra_args: + cmd.extend(extra_args) + + result = exec_command(cmd) + if result["returncode"] != 0: + sys.stderr.write(result["out"] + "\n") + sys.stderr.write(result["err"] + "\n") + env.Exit(1) + + if int(ARGUMENTS.get("PIOVERBOSE", 0)): + print(result["out"]) + print(result["err"]) + + +def get_cmake_code_model(src_dir: Path, build_dir: Path, extra_args=None) -> dict: + + query_file = CMAKE_API_QUERY_DIR / "codemodel-v2" + + if not query_file.exists(): + query_file.parent.mkdir(parents=True, exist_ok=True) + query_file.touch() + + if not is_proper_mbed_ce_project(): + create_default_project_files() + + if is_cmake_reconfigure_required(): + run_cmake(src_dir, build_dir, extra_args) + + if not CMAKE_API_REPLY_DIR.is_dir() or not any(CMAKE_API_REPLY_DIR.iterdir()): + sys.stderr.write("Error: Couldn't find CMake API response file\n") + env.Exit(1) + + codemodel = {} + for target in CMAKE_API_REPLY_DIR.iterdir(): + if target.name.startswith("codemodel-v2"): + with open(target, "r") as fp: + codemodel = json.load(fp) + + assert codemodel["version"]["major"] == 2 + return codemodel + +def get_target_config(project_configs: dict, target_index): + target_json = project_configs.get("targets")[target_index].get("jsonFile", "") + target_config_file = CMAKE_API_REPLY_DIR / target_json + if not target_config_file.is_file(): + sys.stderr.write("Error: Couldn't find target config %s\n" % target_json) + env.Exit(1) + + with open(target_config_file) as fp: + return json.load(fp) + + +def load_target_configurations(cmake_codemodel: dict) -> dict: + configs = {} + project_configs = cmake_codemodel.get("configurations")[0] + for config in project_configs.get("projects", []): + for target_index in config.get("targetIndexes", []): + target_config = get_target_config( + project_configs, target_index + ) + configs[target_config["name"]] = target_config + + return configs + +def generate_project_ld_script() -> pathlib.Path: + + # Run CMake to build the target which generates the linker script + run_cmake(PROJECT_DIR, BUILD_DIR, [ + "--build", + BUILD_DIR, + "--target", + "mbed-linker-script" + ]) + + # Find the linker script. It gets saved in the build dir as + # .link-script.ld. + return next(BUILD_DIR.glob("*.link-script.ld")) + + +def get_targets_by_type(target_configs: dict, target_types: list[str], ignore_targets: list[str] | None=None) -> list: + ignore_targets = ignore_targets or [] + result = [] + for target_config in target_configs.values(): + if ( + target_config["type"] in target_types + and target_config["name"] not in ignore_targets + ): + result.append(target_config) + + return result + +def get_components_map(target_configs: dict, target_types: list[str], ignore_components: list[str] | None=None) -> dict: + result = {} + for config in get_targets_by_type(target_configs, target_types, ignore_components): + if "nameOnDisk" not in config: + config["nameOnDisk"] = "lib%s.a" % config["name"] + result[config["id"]] = {"config": config} + + return result + + +def build_components( + env: Environment, components_map: dict, project_src_dir: pathlib.Path, prepend_dir: pathlib.Path | None = None, debug_allowed=True +): + for k, v in components_map.items(): + components_map[k]["lib"] = build_library( + env, v["config"], project_src_dir, prepend_dir, debug_allowed + ) + + +print("Reading CMake configuration...") +project_codemodel = get_cmake_code_model( + PROJECT_DIR, + BUILD_DIR, + [ + "-DMBED_TARGET=" + get_mbed_target(), + "-DUPLOAD_METHOD=NONE", # Disable Mbed CE upload method system as PlatformIO has its own + ] + + click.parser.split_arg_string(board.get("build.cmake_extra_args", "")), +) + +if not project_codemodel: + sys.stderr.write("Error: Couldn't find code model generated by CMake\n") + env.Exit(1) + +target_configs = load_target_configurations(project_codemodel) + +project_ld_script = generate_project_ld_script() +env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", str(project_ld_script)) + +framework_components_map = get_components_map( + target_configs, + ["STATIC_LIBRARY", "OBJECT_LIBRARY"], + [], +) + +build_components(env, framework_components_map, PROJECT_DIR) + +# project_config = target_configs.get(project_target_name, {}) +# default_config = target_configs.get(default_config_name, {}) +# project_defines = get_app_defines(project_config) +# project_flags = get_app_flags(project_config, default_config) +# link_args = extract_link_args(elf_config) +# app_includes = get_app_includes(elf_config) + +# +# Process main parts of the framework +# + +# libs = find_lib_deps( +# framework_components_map, elf_config, link_args, [project_target_name] +# ) + +# # Extra flags which need to be explicitly specified in LINKFLAGS section because SCons +# # cannot merge them correctly +# extra_flags = filter_args( +# link_args["LINKFLAGS"], +# [ +# "-T", +# "-u", +# "-Wl,--start-group", +# "-Wl,--end-group", +# "-Wl,--whole-archive", +# "-Wl,--no-whole-archive", +# ], +# ) +# link_args["LINKFLAGS"] = sorted(list(set(link_args["LINKFLAGS"]) - set(extra_flags))) + +# +# Main environment configuration +# + +# project_flags.update(link_args) +#env.MergeFlags(project_flags) +env.Prepend( + #CPPPATH=app_includes["plain_includes"], + #CPPDEFINES=project_defines, + #LINKFLAGS=extra_flags, + #LIBS=libs +) \ No newline at end of file diff --git a/tools/python/mbed_platformio/cmake_to_scons_converter.py b/tools/python/mbed_platformio/cmake_to_scons_converter.py new file mode 100644 index 00000000000..ea784fba6ed --- /dev/null +++ b/tools/python/mbed_platformio/cmake_to_scons_converter.py @@ -0,0 +1,119 @@ +""" +Functions for converting build system information from the CMake File API into SCons build targets. +""" + +from __future__ import annotations + +from SCons.Environment import Base as Environment +import pathlib + +def extract_defines(compile_group: dict) -> list[tuple[str, str]]: + def _normalize_define(define_string): + define_string = define_string.strip() + if "=" in define_string: + define, value = define_string.split("=", maxsplit=1) + if any(char in value for char in (' ', '<', '>')): + value = f'"{value}"' + elif '"' in value and not value.startswith("\\"): + value = value.replace('"', '\\"') + return define, value + return define_string + + result = [ + _normalize_define(d.get("define", "")) + for d in compile_group.get("defines", []) if d + ] + + for f in compile_group.get("compileCommandFragments", []): + fragment = f.get("fragment", "").strip() + if fragment.startswith('"'): + fragment = fragment.strip('"') + if fragment.startswith("-D"): + result.append(_normalize_define(fragment[2:])) + + return result + +def prepare_build_envs(config: dict, default_env: Environment, debug_allowed=True): + build_envs = [] + target_compile_groups = config.get("compileGroups", []) + if not target_compile_groups: + print("Warning! The `%s` component doesn't register any source files. " + "Check if sources are set in component's CMakeLists.txt!" % config["name"] + ) + + is_build_type_debug = "debug" in default_env.GetBuildType() and debug_allowed + for cg in target_compile_groups: + includes = [] + sys_includes = [] + for inc in cg.get("includes", []): + inc_path = inc["path"] + if inc.get("isSystem", False): + sys_includes.append(inc_path) + else: + includes.append(inc_path) + + defines = extract_defines(cg) + compile_commands = cg.get("compileCommandFragments", []) + build_env = default_env.Clone() + build_env.SetOption("implicit_cache", 1) + for cc in compile_commands: + build_flags = cc.get("fragment", "").strip("\" ") + if not build_flags.startswith("-D"): + parsed_flags = build_env.ParseFlags(build_flags) + build_env.AppendUnique(**parsed_flags) + if cg.get("language", "") == "ASM": + build_env.AppendUnique(ASPPFLAGS=parsed_flags.get("CCFLAGS", [])) + build_env.AppendUnique(CPPDEFINES=defines, CPPPATH=includes) + if sys_includes: + build_env.Append(CCFLAGS=[("-isystem", inc) for inc in sys_includes]) + build_env.ProcessUnFlags(default_env.get("BUILD_UNFLAGS")) + if is_build_type_debug: + build_env.ConfigureDebugFlags() + build_envs.append(build_env) + + return build_envs + +def compile_source_files( + config: dict, default_env: Environment, project_src_dir: pathlib.Path, prepend_dir: pathlib.Path | None=None, debug_allowed=True +): + build_envs = prepare_build_envs(config, default_env, debug_allowed) + objects = [] + for source in config.get("sources", []): + if source["path"].endswith(".rule"): + continue + compile_group_idx = source.get("compileGroupIndex") + if compile_group_idx is not None: + src_path = pathlib.Path(source.get("path")) + if not src_path.is_absolute(): + # For cases when sources are located near CMakeLists.txt + src_path = project_src_dir / src_path + + obj_dir = pathlib.Path("$BUILD_DIR") / (prepend_dir or "") + if not pathlib.Path(source["path"]).is_absolute(): + obj_path = obj_dir / source["path"] + else: + obj_path = obj_dir / src_path.name + + + objects.append( + build_envs[compile_group_idx].StaticObject( + target=str(obj_path.with_suffix(".o")), + source=str(src_path.resolve()), + ) + ) + + return objects + +def build_library( + default_env: Environment, lib_config: dict, project_src_dir: pathlib.Path, prepend_dir: pathlib.Path | None=None, debug_allowed=True +): + lib_name = lib_config["nameOnDisk"] + lib_path = lib_config["paths"]["build"] + if prepend_dir: + lib_path = prepend_dir / lib_path + lib_objects = compile_source_files( + lib_config, default_env, project_src_dir, prepend_dir, debug_allowed + ) + return default_env.Library( + target=str(pathlib.Path("$BUILD_DIR") / lib_path / lib_name), source=lib_objects + ) diff --git a/tools/python/mbed_platformio/pio_variants.py b/tools/python/mbed_platformio/pio_variants.py new file mode 100644 index 00000000000..afcb6615ab2 --- /dev/null +++ b/tools/python/mbed_platformio/pio_variants.py @@ -0,0 +1,21 @@ + +# Maps PIO variant name to Mbed target name, in the situation where the Mbed target name is different +# from the uppercased version of the variant name +PIO_VARIANT_TO_MBED_TARGET = { + "seeedArchPro": "ARCH_PRO", + "seeedArchMax": "ARCH_MAX", + "frdm_kl25z": "KL25Z", + "frdm_kl43z": "KL43Z", + "frdm_kl46z": "KL46Z", + "frdm_k64f": "K64F", + "frdm_k82f": "K82F", + "IBMEthernetKit": "K64F", + "frdm_k66f": "K66F", + "frdm_k22f": "K22F", + "frdm_kw41z": "KW41Z", + "cloud_jam": "NUCLEO_F401RE", + "cloud_jam_l4": "NUCLEO_L476RG", + "nucleo_h743zi": "NUCLEO_H743ZI2", + "genericSTM32F103RB": "NUCLEO_F103RB", + "disco_h747xi": "DISCO_H747I" +} From f7efa3cd98ed025288c98af5634698c23c56ee69 Mon Sep 17 00:00:00 2001 From: Jamie Smith Date: Sun, 10 Aug 2025 13:14:58 -0700 Subject: [PATCH 02/15] PIO can run CMake configuration! --- tools/python/mbed_platformio/build-mbed-ce.py | 53 +++++++++++-------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/tools/python/mbed_platformio/build-mbed-ce.py b/tools/python/mbed_platformio/build-mbed-ce.py index 8b3c6f5fda0..02fb811984d 100644 --- a/tools/python/mbed_platformio/build-mbed-ce.py +++ b/tools/python/mbed_platformio/build-mbed-ce.py @@ -17,15 +17,12 @@ from platformio.proc import exec_command import click -from .pio_variants import PIO_VARIANT_TO_MBED_TARGET -from .cmake_to_scons_converter import build_library - env: Environment = DefaultEnvironment() platform = env.PioPlatform() board = env.BoardConfig() # Directories -FRAMEWORK_DIR = platform.get_package_dir("framework-mbed-ce") +FRAMEWORK_DIR = Path(platform.get_package_dir("framework-mbed-ce")) BUILD_DIR = Path(env.subst("$BUILD_DIR")) PROJECT_DIR = Path(env.subst("$PROJECT_DIR")) PROJECT_SRC_DIR = Path(env.subst("$PROJECT_SRC_DIR")) @@ -36,6 +33,13 @@ PROJECT_CMAKELISTS_TXT = PROJECT_DIR / "CMakeLists.txt" PROJECT_MBED_APP_JSON5 = PROJECT_DIR / "mbed_app.json5" +# Add mbed-os/tools/python dir to PYTHONPATH so we can import from it. +# This script is run by SCons so it does not have access to any other Python modules by default. +sys.path.append(str(FRAMEWORK_DIR / "tools" / "python")) + +from mbed_platformio.pio_variants import PIO_VARIANT_TO_MBED_TARGET +from mbed_platformio.cmake_to_scons_converter import build_library + def get_mbed_target(): board_type = env.subst("$BOARD") variant = ( @@ -55,6 +59,7 @@ def is_proper_mbed_ce_project(): ) def create_default_project_files(): + print("Mbed CE: Creating default project files") if not PROJECT_CMAKELISTS_TXT.exists(): PROJECT_CMAKELISTS_TXT.write_text( """ # Default CMakeLists.txt for Mbed CE, created by PlatformIO @@ -63,12 +68,12 @@ def create_default_project_files(): set(MBED_APP_JSON_PATH mbed_app.json5) -include(mbed-os/tools/cmake/mbed_toolchain_setup.cmake) +include(${PLATFORMIO_MBED_OS_PATH}/tools/cmake/mbed_toolchain_setup.cmake) project(PlatformIOMbedProject LANGUAGES C CXX ASM) include(mbed_project_setup) -add_subdirectory(mbed-os) +add_subdirectory(${PLATFORMIO_MBED_OS_PATH} mbed-os) """) if not PROJECT_MBED_APP_JSON5.exists(): @@ -113,15 +118,9 @@ def is_cmake_reconfigure_required(): return False -def run_cmake(src_dir: pathlib.Path, build_dir: pathlib.Path, extra_args=None): +def run_cmake(extra_args: list[str] | None = None): cmd = [ str(pathlib.Path(platform.get_package_dir("tool-cmake")) / "bin" / "cmake"), - "-S", - src_dir, - "-B", - build_dir, - "-G", - "Ninja", ] if extra_args: @@ -138,7 +137,7 @@ def run_cmake(src_dir: pathlib.Path, build_dir: pathlib.Path, extra_args=None): print(result["err"]) -def get_cmake_code_model(src_dir: Path, build_dir: Path, extra_args=None) -> dict: +def get_cmake_code_model(cmake_args=None) -> dict: query_file = CMAKE_API_QUERY_DIR / "codemodel-v2" @@ -150,7 +149,8 @@ def get_cmake_code_model(src_dir: Path, build_dir: Path, extra_args=None) -> dic create_default_project_files() if is_cmake_reconfigure_required(): - run_cmake(src_dir, build_dir, extra_args) + print("Mbed CE: Configuring CMake build system...") + run_cmake(cmake_args) if not CMAKE_API_REPLY_DIR.is_dir() or not any(CMAKE_API_REPLY_DIR.iterdir()): sys.stderr.write("Error: Couldn't find CMake API response file\n") @@ -191,7 +191,7 @@ def load_target_configurations(cmake_codemodel: dict) -> dict: def generate_project_ld_script() -> pathlib.Path: # Run CMake to build the target which generates the linker script - run_cmake(PROJECT_DIR, BUILD_DIR, [ + run_cmake([ "--build", BUILD_DIR, "--target", @@ -200,7 +200,7 @@ def generate_project_ld_script() -> pathlib.Path: # Find the linker script. It gets saved in the build dir as # .link-script.ld. - return next(BUILD_DIR.glob("*.link-script.ld")) + return next(BUILD_DIR.glob("*.link_script.ld")) def get_targets_by_type(target_configs: dict, target_types: list[str], ignore_targets: list[str] | None=None) -> list: @@ -234,11 +234,15 @@ def build_components( ) -print("Reading CMake configuration...") project_codemodel = get_cmake_code_model( - PROJECT_DIR, - BUILD_DIR, [ + "-S", + PROJECT_DIR, + "-B", + BUILD_DIR, + "-G", + "Ninja", + "-DPLATFORMIO_MBED_OS_PATH=" + str(FRAMEWORK_DIR), "-DMBED_TARGET=" + get_mbed_target(), "-DUPLOAD_METHOD=NONE", # Disable Mbed CE upload method system as PlatformIO has its own ] @@ -249,11 +253,16 @@ def build_components( sys.stderr.write("Error: Couldn't find code model generated by CMake\n") env.Exit(1) -target_configs = load_target_configurations(project_codemodel) - +print("Generating linker script...") project_ld_script = generate_project_ld_script() env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", str(project_ld_script)) +print("Reading CMake configuration...") +target_configs = load_target_configurations(project_codemodel) + + + + framework_components_map = get_components_map( target_configs, ["STATIC_LIBRARY", "OBJECT_LIBRARY"], From dbf7bc9d18d7def6a0817714c93dfb5e1338b4d2 Mon Sep 17 00:00:00 2001 From: Jamie Smith Date: Fri, 29 Aug 2025 08:47:37 -0700 Subject: [PATCH 03/15] Making progress, able to transfer all flags to SCons! --- tools/python/mbed_platformio/CMakeLists.txt | 22 ++++++ tools/python/mbed_platformio/build-mbed-ce.py | 73 +++++++++---------- .../cmake_to_scons_converter.py | 71 +++++++++++++++++- .../mbed_platformio/dummy_source_file.S | 1 + .../mbed_platformio/dummy_source_file.c | 1 + .../mbed_platformio/dummy_source_file.cpp | 1 + 6 files changed, 128 insertions(+), 41 deletions(-) create mode 100644 tools/python/mbed_platformio/CMakeLists.txt create mode 100644 tools/python/mbed_platformio/dummy_source_file.S create mode 100644 tools/python/mbed_platformio/dummy_source_file.c create mode 100644 tools/python/mbed_platformio/dummy_source_file.cpp diff --git a/tools/python/mbed_platformio/CMakeLists.txt b/tools/python/mbed_platformio/CMakeLists.txt new file mode 100644 index 00000000000..d35d7889533 --- /dev/null +++ b/tools/python/mbed_platformio/CMakeLists.txt @@ -0,0 +1,22 @@ +# +# This file is used as the CMakeLists.txt when building a PlatformIO project. +# + +cmake_minimum_required(VERSION 3.19) +cmake_policy(VERSION 3.19...3.22) + +set(MBED_APP_JSON_PATH mbed_app.json5) + +include(${PLATFORMIO_MBED_OS_PATH}/tools/cmake/mbed_toolchain_setup.cmake) +project(PlatformIOMbedProject + LANGUAGES C CXX ASM) +include(mbed_project_setup) + +add_subdirectory(${PLATFORMIO_MBED_OS_PATH} mbed-os) + +# Dummy executable. Not built, but needed to get the linker flags via the CMake file API +add_executable(PIODummyExecutable + dummy_source_file.c + dummy_source_file.cpp + dummy_source_file.S) +target_link_libraries(PIODummyExecutable mbed-os) \ No newline at end of file diff --git a/tools/python/mbed_platformio/build-mbed-ce.py b/tools/python/mbed_platformio/build-mbed-ce.py index 02fb811984d..062a16aa8de 100644 --- a/tools/python/mbed_platformio/build-mbed-ce.py +++ b/tools/python/mbed_platformio/build-mbed-ce.py @@ -30,7 +30,7 @@ CMAKE_API_QUERY_DIR = CMAKE_API_DIR / "query" CMAKE_API_REPLY_DIR = CMAKE_API_DIR / "reply" -PROJECT_CMAKELISTS_TXT = PROJECT_DIR / "CMakeLists.txt" +PROJECT_CMAKELISTS_TXT = FRAMEWORK_DIR / "tools" / "python" / "mbed_platformio" / "CMakeLists.txt" PROJECT_MBED_APP_JSON5 = PROJECT_DIR / "mbed_app.json5" # Add mbed-os/tools/python dir to PYTHONPATH so we can import from it. @@ -38,7 +38,7 @@ sys.path.append(str(FRAMEWORK_DIR / "tools" / "python")) from mbed_platformio.pio_variants import PIO_VARIANT_TO_MBED_TARGET -from mbed_platformio.cmake_to_scons_converter import build_library +from mbed_platformio.cmake_to_scons_converter import build_library, extract_defines, extract_flags, extract_includes, extract_link_args def get_mbed_target(): board_type = env.subst("$BOARD") @@ -54,28 +54,11 @@ def is_proper_mbed_ce_project(): path.is_file() for path in ( PROJECT_MBED_APP_JSON5, - PROJECT_CMAKELISTS_TXT ) ) def create_default_project_files(): print("Mbed CE: Creating default project files") - if not PROJECT_CMAKELISTS_TXT.exists(): - PROJECT_CMAKELISTS_TXT.write_text( -""" # Default CMakeLists.txt for Mbed CE, created by PlatformIO -cmake_minimum_required(VERSION 3.19) -cmake_policy(VERSION 3.19...3.22) - -set(MBED_APP_JSON_PATH mbed_app.json5) - -include(${PLATFORMIO_MBED_OS_PATH}/tools/cmake/mbed_toolchain_setup.cmake) -project(PlatformIOMbedProject - LANGUAGES C CXX ASM) -include(mbed_project_setup) - -add_subdirectory(${PLATFORMIO_MBED_OS_PATH} mbed-os) -""") - if not PROJECT_MBED_APP_JSON5.exists(): PROJECT_MBED_APP_JSON5.write_text( """ @@ -101,19 +84,21 @@ def is_cmake_reconfigure_required(): ] ninja_buildfile = BUILD_DIR / "build.ninja" - if not CMAKE_API_REPLY_DIR.is_dir() or not not any(CMAKE_API_REPLY_DIR.iterdir()): - return True if not cmake_cache_file.exists(): + print("Mbed CE: Reconfigure required because CMake cache does not exist") + return True + if not CMAKE_API_REPLY_DIR.is_dir() or not any(CMAKE_API_REPLY_DIR.iterdir()): + print("Mbed CE: Reconfigure required because CMake API reply dir is missing") return True if not ninja_buildfile.exists(): + print("Mbed CE: Reconfigure required because Ninja buildfile does not exist") return True cache_file_mtime = cmake_cache_file.stat().st_mtime - if any( - f.stat().st_mtime > cache_file_mtime - for f in cmake_config_files - ): - return True + for file in cmake_config_files: + if file.stat().st_mtime > cache_file_mtime: + print(f"Mbed CE: Reconfigure required because {file.name} was modified") + return True return False @@ -152,6 +137,10 @@ def get_cmake_code_model(cmake_args=None) -> dict: print("Mbed CE: Configuring CMake build system...") run_cmake(cmake_args) + # Seems like CMake doesn't update the timestamp on the cache file if nothing actually changed. + # Ensure that the timestamp is updated so we won't reconfigure next time. + (BUILD_DIR / "CMakeCache.txt").touch() + if not CMAKE_API_REPLY_DIR.is_dir() or not any(CMAKE_API_REPLY_DIR.iterdir()): sys.stderr.write("Error: Couldn't find CMake API response file\n") env.Exit(1) @@ -233,11 +222,13 @@ def build_components( env, v["config"], project_src_dir, prepend_dir, debug_allowed ) +def get_app_defines(app_config: dict): + return extract_defines(app_config["compileGroups"][0]) project_codemodel = get_cmake_code_model( [ "-S", - PROJECT_DIR, + PROJECT_CMAKELISTS_TXT.parent, "-B", BUILD_DIR, "-G", @@ -260,9 +251,6 @@ def build_components( print("Reading CMake configuration...") target_configs = load_target_configurations(project_codemodel) - - - framework_components_map = get_components_map( target_configs, ["STATIC_LIBRARY", "OBJECT_LIBRARY"], @@ -271,12 +259,17 @@ def build_components( build_components(env, framework_components_map, PROJECT_DIR) -# project_config = target_configs.get(project_target_name, {}) -# default_config = target_configs.get(default_config_name, {}) -# project_defines = get_app_defines(project_config) -# project_flags = get_app_flags(project_config, default_config) -# link_args = extract_link_args(elf_config) -# app_includes = get_app_includes(elf_config) +mbed_os_lib_target_json = target_configs.get("mbed-os", {}) +app_target_json = target_configs.get("PIODummyExecutable", {}) +project_defines = get_app_defines(app_target_json) +project_flags = extract_flags(app_target_json) +link_args = extract_link_args(app_target_json) +app_includes = extract_includes(app_target_json) +print(f"link_args={link_args!r}") + +# The CMake build system adds a flag in mbed_set_post_build() to output a map file. +# We need to do that here. +link_args["LINKFLAGS"].append(f"-Wl,-Map={str(BUILD_DIR / 'firmware.map')}") # # Process main parts of the framework @@ -305,11 +298,11 @@ def build_components( # Main environment configuration # -# project_flags.update(link_args) -#env.MergeFlags(project_flags) +project_flags.update(link_args) +env.MergeFlags(project_flags) env.Prepend( - #CPPPATH=app_includes["plain_includes"], - #CPPDEFINES=project_defines, + CPPPATH=app_includes["plain_includes"], + CPPDEFINES=project_defines, #LINKFLAGS=extra_flags, #LIBS=libs ) \ No newline at end of file diff --git a/tools/python/mbed_platformio/cmake_to_scons_converter.py b/tools/python/mbed_platformio/cmake_to_scons_converter.py index ea784fba6ed..da62e62eff7 100644 --- a/tools/python/mbed_platformio/cmake_to_scons_converter.py +++ b/tools/python/mbed_platformio/cmake_to_scons_converter.py @@ -6,6 +6,7 @@ from SCons.Environment import Base as Environment import pathlib +import click def extract_defines(compile_group: dict) -> list[tuple[str, str]]: def _normalize_define(define_string): @@ -33,7 +34,10 @@ def _normalize_define(define_string): return result -def prepare_build_envs(config: dict, default_env: Environment, debug_allowed=True): +def prepare_build_envs(config: dict, default_env: Environment, debug_allowed=True) -> list[Environment]: + """ + Creates the Scons Environment(s) needed to build the source files in a CMake target + """ build_envs = [] target_compile_groups = config.get("compileGroups", []) if not target_compile_groups: @@ -76,6 +80,9 @@ def prepare_build_envs(config: dict, default_env: Environment, debug_allowed=Tru def compile_source_files( config: dict, default_env: Environment, project_src_dir: pathlib.Path, prepend_dir: pathlib.Path | None=None, debug_allowed=True ): + """ + Generates SCons rules to compile the source files in a target + """ build_envs = prepare_build_envs(config, default_env, debug_allowed) objects = [] for source in config.get("sources", []): @@ -117,3 +124,65 @@ def build_library( return default_env.Library( target=str(pathlib.Path("$BUILD_DIR") / lib_path / lib_name), source=lib_objects ) + +def extract_flags(target_json: dict) -> dict[str, list[str]]: + """ + Returns a dictionary with flags for SCons based on a given CMake target + """ + def _extract_flags(target_json: dict) -> dict[str, list[str]]: + flags = {} + for cg in target_json["compileGroups"]: + flags[cg["language"]] = [] + for ccfragment in cg["compileCommandFragments"]: + fragment = ccfragment.get("fragment", "").strip("\" ") + if not fragment or fragment.startswith("-D"): + continue + flags[cg["language"]].extend( + click.parser.split_arg_string(fragment.strip()) + ) + + return flags + + default_flags = _extract_flags(target_json) + + # Flags are sorted because CMake randomly populates build flags in code model + return { + "ASPPFLAGS": default_flags.get("ASM"), + "CFLAGS": default_flags.get("C"), + "CXXFLAGS": default_flags.get("CXX"), + } + + +def extract_includes(target_json: dict) -> dict[str, list[str]]: + """ + Extract the includes from a CMake target and return an SCons-style dict + """ + plain_includes = [] + sys_includes = [] + cg = target_json["compileGroups"][0] + for inc in cg.get("includes", []): + inc_path = inc["path"] + if inc.get("isSystem", False): + sys_includes.append(inc_path) + else: + plain_includes.append(inc_path) + + return {"plain_includes": plain_includes, "sys_includes": sys_includes} + +def extract_link_args(target_json: dict) -> dict[str, list[str]]: + """ + Extract the linker flags from a CMake target and return an SCons-style dict + """ + + link_args = {"LINKFLAGS": []} + + for f in target_json.get("link", {}).get("commandFragments", []): + fragment = f.get("fragment", "").strip() + fragment_role = f.get("role", "").strip() + if not fragment or not fragment_role: + continue + args = click.parser.split_arg_string(fragment) + if fragment_role == "flags": + link_args["LINKFLAGS"].extend(args) + + return link_args \ No newline at end of file diff --git a/tools/python/mbed_platformio/dummy_source_file.S b/tools/python/mbed_platformio/dummy_source_file.S new file mode 100644 index 00000000000..cfd852831b2 --- /dev/null +++ b/tools/python/mbed_platformio/dummy_source_file.S @@ -0,0 +1 @@ +// Empty C++ source file so that we can scan the ASM flags via the CMake API \ No newline at end of file diff --git a/tools/python/mbed_platformio/dummy_source_file.c b/tools/python/mbed_platformio/dummy_source_file.c new file mode 100644 index 00000000000..c0b97ea8116 --- /dev/null +++ b/tools/python/mbed_platformio/dummy_source_file.c @@ -0,0 +1 @@ +// Empty C++ source file so that we can scan the C flags via the CMake API \ No newline at end of file diff --git a/tools/python/mbed_platformio/dummy_source_file.cpp b/tools/python/mbed_platformio/dummy_source_file.cpp new file mode 100644 index 00000000000..a3b324ab247 --- /dev/null +++ b/tools/python/mbed_platformio/dummy_source_file.cpp @@ -0,0 +1 @@ +// Empty C++ source file so that we can scan the C++ flags via the CMake API \ No newline at end of file From 34f6fb638e2603920e2549bdb37be790ab022561 Mon Sep 17 00:00:00 2001 From: Jamie Smith Date: Sat, 6 Sep 2025 09:26:46 -0700 Subject: [PATCH 04/15] Working!! --- tools/python/mbed_platformio/build-mbed-ce.py | 45 ++++++---------- .../cmake_to_scons_converter.py | 53 ++++++++++--------- 2 files changed, 43 insertions(+), 55 deletions(-) diff --git a/tools/python/mbed_platformio/build-mbed-ce.py b/tools/python/mbed_platformio/build-mbed-ce.py index 062a16aa8de..c208a59d772 100644 --- a/tools/python/mbed_platformio/build-mbed-ce.py +++ b/tools/python/mbed_platformio/build-mbed-ce.py @@ -244,11 +244,12 @@ def get_app_defines(app_config: dict): sys.stderr.write("Error: Couldn't find code model generated by CMake\n") env.Exit(1) -print("Generating linker script...") +print("Mbed CE: Generating linker script...") project_ld_script = generate_project_ld_script() env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", str(project_ld_script)) +env.Append(LDSCRIPT_PATH=str(project_ld_script)) -print("Reading CMake configuration...") +print("Mbed CE: Reading CMake configuration...") target_configs = load_target_configurations(project_codemodel) framework_components_map = get_components_map( @@ -265,44 +266,30 @@ def get_app_defines(app_config: dict): project_flags = extract_flags(app_target_json) link_args = extract_link_args(app_target_json) app_includes = extract_includes(app_target_json) -print(f"link_args={link_args!r}") # The CMake build system adds a flag in mbed_set_post_build() to output a map file. # We need to do that here. -link_args["LINKFLAGS"].append(f"-Wl,-Map={str(BUILD_DIR / 'firmware.map')}") +map_file = BUILD_DIR / 'firmware.map' +link_args["LINKFLAGS"].append(f"-Wl,-Map={str(map_file)}") -# -# Process main parts of the framework -# +# Link the main Mbed OS library using -Wl,--whole-archive. This is needed for the resolution of weak symbols +# within this archive. I found this workaround here on the mailing list: +# https://scons-users.scons.narkive.com/noJni4Ut/workaround-support-for-wl-whole-archive +# However, for some reason SCons need to think that the flag is "in the source dir" or it appends a path +# to the build dir before it. So, I added the $PROJECT_DIR part. +whole_archive = env.Command('$PROJECT_DIR/-Wl,--whole-archive', [], '') +no_whole_archive = env.Command('$PROJECT_DIR/-Wl,--no-whole-archive', [], '') -# libs = find_lib_deps( -# framework_components_map, elf_config, link_args, [project_target_name] -# ) - -# # Extra flags which need to be explicitly specified in LINKFLAGS section because SCons -# # cannot merge them correctly -# extra_flags = filter_args( -# link_args["LINKFLAGS"], -# [ -# "-T", -# "-u", -# "-Wl,--start-group", -# "-Wl,--end-group", -# "-Wl,--whole-archive", -# "-Wl,--no-whole-archive", -# ], -# ) -# link_args["LINKFLAGS"] = sorted(list(set(link_args["LINKFLAGS"]) - set(extra_flags))) +# Add the Mbed OS archive and the link flags at the end of the linker command line after all the app source files +env.Append(PIOAPPENDBUILDFILES=[whole_archive, "$BUILD_DIR\\mbed-os\\libmbed-os.a", no_whole_archive]) # # Main environment configuration # - project_flags.update(link_args) env.MergeFlags(project_flags) env.Prepend( CPPPATH=app_includes["plain_includes"], CPPDEFINES=project_defines, - #LINKFLAGS=extra_flags, - #LIBS=libs -) \ No newline at end of file +) + diff --git a/tools/python/mbed_platformio/cmake_to_scons_converter.py b/tools/python/mbed_platformio/cmake_to_scons_converter.py index da62e62eff7..728071f5535 100644 --- a/tools/python/mbed_platformio/cmake_to_scons_converter.py +++ b/tools/python/mbed_platformio/cmake_to_scons_converter.py @@ -4,6 +4,8 @@ from __future__ import annotations +import collections + from SCons.Environment import Base as Environment import pathlib import click @@ -34,15 +36,15 @@ def _normalize_define(define_string): return result -def prepare_build_envs(config: dict, default_env: Environment, debug_allowed=True) -> list[Environment]: +def prepare_build_envs(target_json: dict, default_env: Environment, debug_allowed=True) -> list[Environment]: """ Creates the Scons Environment(s) needed to build the source files in a CMake target """ build_envs = [] - target_compile_groups = config.get("compileGroups", []) + target_compile_groups = target_json.get("compileGroups", []) if not target_compile_groups: print("Warning! The `%s` component doesn't register any source files. " - "Check if sources are set in component's CMakeLists.txt!" % config["name"] + "Check if sources are set in component's CMakeLists.txt!" % target_json["name"] ) is_build_type_debug = "debug" in default_env.GetBuildType() and debug_allowed @@ -57,16 +59,10 @@ def prepare_build_envs(config: dict, default_env: Environment, debug_allowed=Tru includes.append(inc_path) defines = extract_defines(cg) - compile_commands = cg.get("compileCommandFragments", []) + flags = extract_flags(target_json) build_env = default_env.Clone() build_env.SetOption("implicit_cache", 1) - for cc in compile_commands: - build_flags = cc.get("fragment", "").strip("\" ") - if not build_flags.startswith("-D"): - parsed_flags = build_env.ParseFlags(build_flags) - build_env.AppendUnique(**parsed_flags) - if cg.get("language", "") == "ASM": - build_env.AppendUnique(ASPPFLAGS=parsed_flags.get("CCFLAGS", [])) + build_env.MergeFlags(flags) build_env.AppendUnique(CPPDEFINES=defines, CPPPATH=includes) if sys_includes: build_env.Append(CCFLAGS=[("-isystem", inc) for inc in sys_includes]) @@ -121,29 +117,34 @@ def build_library( lib_objects = compile_source_files( lib_config, default_env, project_src_dir, prepend_dir, debug_allowed ) + + #print(f"Created build rule for " + str(pathlib.Path("$BUILD_DIR") / lib_path / lib_name)) + return default_env.Library( target=str(pathlib.Path("$BUILD_DIR") / lib_path / lib_name), source=lib_objects ) +def _get_flags_for_compile_group(compile_group_json: dict) -> list[str]: + """ + Extract the flags from a CMake compile group. + """ + flags = [] + for ccfragment in compile_group_json["compileCommandFragments"]: + fragment = ccfragment.get("fragment", "").strip("\" ") + if not fragment or fragment.startswith("-D"): + continue + flags.extend( + click.parser.split_arg_string(fragment.strip()) + ) + return flags + def extract_flags(target_json: dict) -> dict[str, list[str]]: """ Returns a dictionary with flags for SCons based on a given CMake target """ - def _extract_flags(target_json: dict) -> dict[str, list[str]]: - flags = {} - for cg in target_json["compileGroups"]: - flags[cg["language"]] = [] - for ccfragment in cg["compileCommandFragments"]: - fragment = ccfragment.get("fragment", "").strip("\" ") - if not fragment or fragment.startswith("-D"): - continue - flags[cg["language"]].extend( - click.parser.split_arg_string(fragment.strip()) - ) - - return flags - - default_flags = _extract_flags(target_json) + default_flags = collections.defaultdict(list) + for cg in target_json["compileGroups"]: + default_flags[cg["language"]].extend(_get_flags_for_compile_group(cg)) # Flags are sorted because CMake randomly populates build flags in code model return { From c4f382e16faba042e2db1425767c7cf93de57610 Mon Sep 17 00:00:00 2001 From: Jamie Smith Date: Sat, 6 Sep 2025 10:11:14 -0700 Subject: [PATCH 05/15] Fix clean build issue --- tools/python/mbed_platformio/build-mbed-ce.py | 47 ++++++++++--------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/tools/python/mbed_platformio/build-mbed-ce.py b/tools/python/mbed_platformio/build-mbed-ce.py index c208a59d772..ca1c9359896 100644 --- a/tools/python/mbed_platformio/build-mbed-ce.py +++ b/tools/python/mbed_platformio/build-mbed-ce.py @@ -10,7 +10,6 @@ from pathlib import Path import json import sys -import os from SCons.Script import DefaultEnvironment, ARGUMENTS from SCons.Environment import Base as Environment @@ -103,15 +102,8 @@ def is_cmake_reconfigure_required(): return False -def run_cmake(extra_args: list[str] | None = None): - cmd = [ - str(pathlib.Path(platform.get_package_dir("tool-cmake")) / "bin" / "cmake"), - ] - - if extra_args: - cmd.extend(extra_args) - - result = exec_command(cmd) +def run_tool(command_and_args: list[str] | None = None): + result = exec_command(command_and_args) if result["returncode"] != 0: sys.stderr.write(result["out"] + "\n") sys.stderr.write(result["err"] + "\n") @@ -122,7 +114,7 @@ def run_cmake(extra_args: list[str] | None = None): print(result["err"]) -def get_cmake_code_model(cmake_args=None) -> dict: +def get_cmake_code_model(cmake_args: list) -> dict: query_file = CMAKE_API_QUERY_DIR / "codemodel-v2" @@ -135,7 +127,8 @@ def get_cmake_code_model(cmake_args=None) -> dict: if is_cmake_reconfigure_required(): print("Mbed CE: Configuring CMake build system...") - run_cmake(cmake_args) + cmake_command = [str(pathlib.Path(platform.get_package_dir("tool-cmake")) / "bin" / "cmake"), *cmake_args] + run_tool(cmake_command) # Seems like CMake doesn't update the timestamp on the cache file if nothing actually changed. # Ensure that the timestamp is updated so we won't reconfigure next time. @@ -179,13 +172,16 @@ def load_target_configurations(cmake_codemodel: dict) -> dict: def generate_project_ld_script() -> pathlib.Path: - # Run CMake to build the target which generates the linker script - run_cmake([ - "--build", - BUILD_DIR, - "--target", + # Run Ninja to build the target which generates the linker script. + # Note that we don't want to use CMake as running it has the side effect of redoing + # the file API query. + cmd = [ + str(pathlib.Path(platform.get_package_dir("tool-ninja")) / "ninja"), + "-C", + str(BUILD_DIR), "mbed-linker-script" - ]) + ] + run_tool(cmd) # Find the linker script. It gets saved in the build dir as # .link-script.ld. @@ -233,6 +229,7 @@ def get_app_defines(app_config: dict): BUILD_DIR, "-G", "Ninja", + "-DCMAKE_MAKE_PROGRAM=" + str(pathlib.Path(platform.get_package_dir("tool-ninja")) / "ninja"), "-DPLATFORMIO_MBED_OS_PATH=" + str(FRAMEWORK_DIR), "-DMBED_TARGET=" + get_mbed_target(), "-DUPLOAD_METHOD=NONE", # Disable Mbed CE upload method system as PlatformIO has its own @@ -244,11 +241,6 @@ def get_app_defines(app_config: dict): sys.stderr.write("Error: Couldn't find code model generated by CMake\n") env.Exit(1) -print("Mbed CE: Generating linker script...") -project_ld_script = generate_project_ld_script() -env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", str(project_ld_script)) -env.Append(LDSCRIPT_PATH=str(project_ld_script)) - print("Mbed CE: Reading CMake configuration...") target_configs = load_target_configurations(project_codemodel) @@ -293,3 +285,12 @@ def get_app_defines(app_config: dict): CPPDEFINES=project_defines, ) +# Run Ninja to produce the linker script. +# Note that this seems to execute CMake, causing the code model query to be re-done. +# So, we have to do this after we are done using the results of said query. +print("Mbed CE: Generating linker script...") +project_ld_script = generate_project_ld_script() +env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", str(project_ld_script)) +env.Append(LDSCRIPT_PATH=str(project_ld_script)) + +print("Mbed CE: Build environment configured.") \ No newline at end of file From f885b9e3ff1d381569f556e61c3683667c884902 Mon Sep 17 00:00:00 2001 From: Jamie Smith Date: Sat, 6 Sep 2025 10:37:12 -0700 Subject: [PATCH 06/15] Support 'pio debug' --- tools/python/mbed_platformio/build-mbed-ce.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/python/mbed_platformio/build-mbed-ce.py b/tools/python/mbed_platformio/build-mbed-ce.py index ca1c9359896..2b57c3662b5 100644 --- a/tools/python/mbed_platformio/build-mbed-ce.py +++ b/tools/python/mbed_platformio/build-mbed-ce.py @@ -221,6 +221,7 @@ def build_components( def get_app_defines(app_config: dict): return extract_defines(app_config["compileGroups"][0]) +build_type = "Debug" if ("debug" in env.GetBuildType()) else "Release" project_codemodel = get_cmake_code_model( [ "-S", @@ -230,6 +231,7 @@ def get_app_defines(app_config: dict): "-G", "Ninja", "-DCMAKE_MAKE_PROGRAM=" + str(pathlib.Path(platform.get_package_dir("tool-ninja")) / "ninja"), + "-DCMAKE_BUILD_TYPE=" + build_type, "-DPLATFORMIO_MBED_OS_PATH=" + str(FRAMEWORK_DIR), "-DMBED_TARGET=" + get_mbed_target(), "-DUPLOAD_METHOD=NONE", # Disable Mbed CE upload method system as PlatformIO has its own From 07c5a211e4d788e07f1f437c7b82c8ceed0b8503 Mon Sep 17 00:00:00 2001 From: Jamie Smith Date: Sat, 6 Sep 2025 12:11:13 -0700 Subject: [PATCH 07/15] Fix mbed_app.json path being specified wrong --- tools/cmake/mbed_generate_configuration.cmake | 54 +++++++++++++++-- tools/python/mbed_platformio/CMakeLists.txt | 3 +- tools/python/mbed_platformio/build-mbed-ce.py | 33 +++++++--- .../cmake_to_scons_converter.py | 60 ++++++++++++------- 4 files changed, 111 insertions(+), 39 deletions(-) diff --git a/tools/cmake/mbed_generate_configuration.cmake b/tools/cmake/mbed_generate_configuration.cmake index 19a788aa19a..cd3e5eb1b0f 100644 --- a/tools/cmake/mbed_generate_configuration.cmake +++ b/tools/cmake/mbed_generate_configuration.cmake @@ -11,6 +11,30 @@ set(MBED_NEED_TO_RECONFIGURE FALSE) +# Check that path variables (MBED_APP_JSON_PATH, CUSTOM_TARGETS_JSON_PATH) are valid and set +# vars (HAS_CUSTOM_TARGETS_JSON, HAS_MBED_APP_JSON) based on whether they exist. +# Also, convert all relative paths to absolute paths, rooted at CMAKE_SOURCE_DIR. +# This makes sure that they are interpreted the same way everywhere. +foreach(json_var_name MBED_APP_JSON CUSTOM_TARGETS_JSON) + + if("${${json_var_name}_PATH}" STREQUAL "") + set(HAS_${json_var_name} FALSE) + else() + get_filename_component(${json_var_name}_PATH "${${json_var_name}_PATH}" ABSOLUTE BASE_DIR ${CMAKE_SOURCE_DIR}) + if(NOT EXISTS ${${json_var_name}_PATH} OR IS_DIRECTORY ${${json_var_name}_PATH}) + message(FATAL_ERROR "${json_var_name}_PATH value of ${${json_var_name}_PATH} is not a valid file!") + endif() + set(HAS_${json_var_name} TRUE) + endif() +endforeach() + +if("${CUSTOM_TARGETS_JSON_PATH}" STREQUAL "") + set(HAS_CUSTOM_TARGETS_JSON FALSE) +else() + set(HAS_CUSTOM_TARGETS_JSON TRUE) + get_filename_component(CUSTOM_TARGETS_JSON_PATH "${CUSTOM_TARGETS_JSON_PATH}" ABSOLUTE BASE_DIR ${CMAKE_SOURCE_DIR}) +endif() + # First, verify that MBED_TARGET has not changed if(DEFINED MBED_INTERNAL_LAST_MBED_TARGET) if(NOT "${MBED_INTERNAL_LAST_MBED_TARGET}" STREQUAL "${MBED_TARGET}") @@ -42,9 +66,19 @@ endif() if(NOT MBED_NEED_TO_RECONFIGURE) file(TIMESTAMP ${CMAKE_CURRENT_BINARY_DIR}/mbed_config.cmake MBED_CONFIG_CMAKE_TIMESTAMP "%s" UTC) + set(MBED_APP_JSON_FOUND FALSE) + set(CUSTOM_TARGETS_JSON_FOUND FALSE) + foreach(CONFIG_JSON ${MBED_CONFIG_JSON_SOURCE_FILES}) get_filename_component(CONFIG_JSON_ABSPATH ${CONFIG_JSON} ABSOLUTE) + if(CONFIG_JSON_ABSPATH STREQUAL MBED_APP_JSON_PATH) + set(MBED_APP_JSON_FOUND TRUE) + endif() + if(CONFIG_JSON_ABSPATH STREQUAL CUSTOM_TARGETS_JSON_PATH) + set(CUSTOM_TARGETS_JSON_FOUND TRUE) + endif() + if(NOT EXISTS ${CONFIG_JSON_ABSPATH}) message(STATUS "Mbed: ${CONFIG_JSON} deleted or renamed, regenerating configs...") set(MBED_NEED_TO_RECONFIGURE TRUE) @@ -60,20 +94,28 @@ if(NOT MBED_NEED_TO_RECONFIGURE) endforeach() endif() -# Convert all relative paths to absolute paths, rooted at CMAKE_SOURCE_DIR. -# This makes sure that they are interpreted the same way everywhere. -get_filename_component(MBED_APP_JSON_PATH "${MBED_APP_JSON_PATH}" ABSOLUTE BASE_DIR ${CMAKE_SOURCE_DIR}) -get_filename_component(CUSTOM_TARGETS_JSON_PATH "${CUSTOM_TARGETS_JSON_PATH}" ABSOLUTE BASE_DIR ${CMAKE_SOURCE_DIR}) +if(NOT MBED_NEED_TO_RECONFIGURE) + # Corner case: if we previously had not set an mbed_app.json and now we do, we need to detect that + # and reconfigure. + if(HAS_MBED_APP_JSON AND NOT MBED_APP_JSON_FOUND) + message(STATUS "Mbed: mbed_app.json added/moved, regenerating configs...") + set(MBED_NEED_TO_RECONFIGURE TRUE) + endif() + if(HAS_CUSTOM_TARGETS_JSON AND NOT CUSTOM_TARGETS_JSON_FOUND) + message(STATUS "Mbed: custom_targets.json added/moved, regenerating configs...") + set(MBED_NEED_TO_RECONFIGURE TRUE) + endif() +endif() if(MBED_NEED_TO_RECONFIGURE) # Generate mbed_config.cmake for this target - if(EXISTS "${MBED_APP_JSON_PATH}" AND (NOT IS_DIRECTORY "${MBED_APP_JSON_PATH}")) + if(HAS_MBED_APP_JSON) set(APP_CONFIG_ARGUMENT --app-config "${MBED_APP_JSON_PATH}") else() set(APP_CONFIG_ARGUMENT "") endif() - if(EXISTS "${CUSTOM_TARGETS_JSON_PATH}" AND (NOT IS_DIRECTORY "${CUSTOM_TARGETS_JSON_PATH}")) + if(HAS_CUSTOM_TARGETS_JSON) set(CUSTOM_TARGET_ARGUMENT --custom-targets-json "${CUSTOM_TARGETS_JSON_PATH}") else() set(CUSTOM_TARGET_ARGUMENT "") diff --git a/tools/python/mbed_platformio/CMakeLists.txt b/tools/python/mbed_platformio/CMakeLists.txt index d35d7889533..73b21616cda 100644 --- a/tools/python/mbed_platformio/CMakeLists.txt +++ b/tools/python/mbed_platformio/CMakeLists.txt @@ -5,7 +5,8 @@ cmake_minimum_required(VERSION 3.19) cmake_policy(VERSION 3.19...3.22) -set(MBED_APP_JSON_PATH mbed_app.json5) +set(MBED_APP_JSON_PATH ${PLATFORMIO_PROJECT_PATH}/mbed_app.json5) +message("MBED_APP_JSON_PATH = ${MBED_APP_JSON_PATH}") include(${PLATFORMIO_MBED_OS_PATH}/tools/cmake/mbed_toolchain_setup.cmake) project(PlatformIOMbedProject diff --git a/tools/python/mbed_platformio/build-mbed-ce.py b/tools/python/mbed_platformio/build-mbed-ce.py index 2b57c3662b5..e45809033e5 100644 --- a/tools/python/mbed_platformio/build-mbed-ce.py +++ b/tools/python/mbed_platformio/build-mbed-ce.py @@ -31,13 +31,19 @@ PROJECT_CMAKELISTS_TXT = FRAMEWORK_DIR / "tools" / "python" / "mbed_platformio" / "CMakeLists.txt" PROJECT_MBED_APP_JSON5 = PROJECT_DIR / "mbed_app.json5" +PROJECT_TARGET_CONFIG_H = BUILD_DIR / "mbed-os" / "generated-headers" / "mbed-target-config.h" + +CMAKE_BUILD_TYPE = "Debug" if ("debug" in env.GetBuildType()) else "Release" + +NINJA_PATH = pathlib.Path(platform.get_package_dir("tool-ninja")) / "ninja" +CMAKE_PATH = pathlib.Path(platform.get_package_dir("tool-cmake")) / "bin" / "cmake" # Add mbed-os/tools/python dir to PYTHONPATH so we can import from it. # This script is run by SCons so it does not have access to any other Python modules by default. sys.path.append(str(FRAMEWORK_DIR / "tools" / "python")) from mbed_platformio.pio_variants import PIO_VARIANT_TO_MBED_TARGET -from mbed_platformio.cmake_to_scons_converter import build_library, extract_defines, extract_flags, extract_includes, extract_link_args +from mbed_platformio.cmake_to_scons_converter import build_library, extract_defines, extract_flags, extract_includes, extract_link_args, find_included_files def get_mbed_target(): board_type = env.subst("$BOARD") @@ -64,7 +70,7 @@ def create_default_project_files(): { "target_overrides": { "*": { - "platform.stdio-baud-rate": 115200, + "platform.stdio-baud-rate": 9600, // matches PlatformIO default "platform.stdio-buffered-serial": 1, // Uncomment to use mbed-baremetal instead of mbed-os @@ -93,6 +99,11 @@ def is_cmake_reconfigure_required(): print("Mbed CE: Reconfigure required because Ninja buildfile does not exist") return True + # If the JSON files have 'Debug' in their names that means this project was previously configured as Debug. + if not any(CMAKE_API_REPLY_DIR.glob(f"directory-*{CMAKE_BUILD_TYPE}*.json")): + print("Mbed CE: Reconfigure required because build type (debug / release) changed.") + return True + cache_file_mtime = cmake_cache_file.stat().st_mtime for file in cmake_config_files: if file.stat().st_mtime > cache_file_mtime: @@ -127,7 +138,7 @@ def get_cmake_code_model(cmake_args: list) -> dict: if is_cmake_reconfigure_required(): print("Mbed CE: Configuring CMake build system...") - cmake_command = [str(pathlib.Path(platform.get_package_dir("tool-cmake")) / "bin" / "cmake"), *cmake_args] + cmake_command = [str(CMAKE_PATH), *cmake_args] run_tool(cmake_command) # Seems like CMake doesn't update the timestamp on the cache file if nothing actually changed. @@ -211,17 +222,16 @@ def get_components_map(target_configs: dict, target_types: list[str], ignore_com def build_components( - env: Environment, components_map: dict, project_src_dir: pathlib.Path, prepend_dir: pathlib.Path | None = None, debug_allowed=True + env: Environment, components_map: dict, project_src_dir: pathlib.Path ): for k, v in components_map.items(): components_map[k]["lib"] = build_library( - env, v["config"], project_src_dir, prepend_dir, debug_allowed + env, v["config"], project_src_dir, FRAMEWORK_DIR, pathlib.Path("$BUILD_DIR/mbed-os") ) def get_app_defines(app_config: dict): return extract_defines(app_config["compileGroups"][0]) -build_type = "Debug" if ("debug" in env.GetBuildType()) else "Release" project_codemodel = get_cmake_code_model( [ "-S", @@ -230,9 +240,10 @@ def get_app_defines(app_config: dict): BUILD_DIR, "-G", "Ninja", - "-DCMAKE_MAKE_PROGRAM=" + str(pathlib.Path(platform.get_package_dir("tool-ninja")) / "ninja"), - "-DCMAKE_BUILD_TYPE=" + build_type, - "-DPLATFORMIO_MBED_OS_PATH=" + str(FRAMEWORK_DIR), + "-DCMAKE_MAKE_PROGRAM=" + str(NINJA_PATH.as_posix()), # Note: CMake prefers to be passed paths with forward slashes, so use as_posix() + "-DCMAKE_BUILD_TYPE=" + CMAKE_BUILD_TYPE, + "-DPLATFORMIO_MBED_OS_PATH=" + str(FRAMEWORK_DIR.as_posix()), + "-DPLATFORMIO_PROJECT_PATH=" + str(PROJECT_DIR.as_posix()), "-DMBED_TARGET=" + get_mbed_target(), "-DUPLOAD_METHOD=NONE", # Disable Mbed CE upload method system as PlatformIO has its own ] @@ -287,6 +298,10 @@ def get_app_defines(app_config: dict): CPPDEFINES=project_defines, ) +# Set up a dependency between all application source files and mbed-target-config.h. +# This ensures that the app will be recompiled if the header changes. +env.Append(PIO_EXTRA_APP_SOURCE_DEPS=find_included_files(env)) + # Run Ninja to produce the linker script. # Note that this seems to execute CMake, causing the code model query to be re-done. # So, we have to do this after we are done using the results of said query. diff --git a/tools/python/mbed_platformio/cmake_to_scons_converter.py b/tools/python/mbed_platformio/cmake_to_scons_converter.py index 728071f5535..37e4c75501f 100644 --- a/tools/python/mbed_platformio/cmake_to_scons_converter.py +++ b/tools/python/mbed_platformio/cmake_to_scons_converter.py @@ -36,7 +36,7 @@ def _normalize_define(define_string): return result -def prepare_build_envs(target_json: dict, default_env: Environment, debug_allowed=True) -> list[Environment]: +def prepare_build_envs(target_json: dict, default_env: Environment) -> list[Environment]: """ Creates the Scons Environment(s) needed to build the source files in a CMake target """ @@ -47,7 +47,6 @@ def prepare_build_envs(target_json: dict, default_env: Environment, debug_allowe "Check if sources are set in component's CMakeLists.txt!" % target_json["name"] ) - is_build_type_debug = "debug" in default_env.GetBuildType() and debug_allowed for cg in target_compile_groups: includes = [] sys_includes = [] @@ -67,55 +66,59 @@ def prepare_build_envs(target_json: dict, default_env: Environment, debug_allowe if sys_includes: build_env.Append(CCFLAGS=[("-isystem", inc) for inc in sys_includes]) build_env.ProcessUnFlags(default_env.get("BUILD_UNFLAGS")) - if is_build_type_debug: - build_env.ConfigureDebugFlags() build_envs.append(build_env) return build_envs def compile_source_files( - config: dict, default_env: Environment, project_src_dir: pathlib.Path, prepend_dir: pathlib.Path | None=None, debug_allowed=True -): + config: dict, default_env: Environment, project_src_dir: pathlib.Path, framework_dir: pathlib.Path, framework_obj_dir: pathlib.Path) -> list: """ - Generates SCons rules to compile the source files in a target + Generates SCons rules to compile the source files in a target. + Returns list of object files to build. + + :param framework_dir: Path to the Mbed CE framework source + :param framework_obj_dir: Path to the directory where object files for Mbed CE will be saved. """ - build_envs = prepare_build_envs(config, default_env, debug_allowed) + build_envs = prepare_build_envs(config, default_env) objects = [] for source in config.get("sources", []): if source["path"].endswith(".rule"): continue compile_group_idx = source.get("compileGroupIndex") if compile_group_idx is not None: + + # Get absolute path to source, resolving relative to source dir if needed src_path = pathlib.Path(source.get("path")) if not src_path.is_absolute(): - # For cases when sources are located near CMakeLists.txt src_path = project_src_dir / src_path - obj_dir = pathlib.Path("$BUILD_DIR") / (prepend_dir or "") - if not pathlib.Path(source["path"]).is_absolute(): - obj_path = obj_dir / source["path"] + # Figure out object path + if src_path.is_relative_to(project_src_dir): + obj_path = (pathlib.Path("$BUILD_DIR") / src_path.relative_to(project_src_dir)).with_suffix(".o") + elif src_path.is_relative_to(framework_dir): + obj_path = (framework_obj_dir / src_path.relative_to(framework_dir)).with_suffix(".o") else: - obj_path = obj_dir / src_path.name + raise RuntimeError(f"Source path {src_path!s} outside of project source dir and framework dir, don't know where to save object file!") + + env = build_envs[compile_group_idx] + objects.append(env.StaticObject(target=str(obj_path), source=str(src_path))) + + # SCons isn't smart enough to add a dependency based on the "-include" compiler flag, so + # manually add one. + for included_file in find_included_files(env): + env.Depends(str(obj_path), included_file) - objects.append( - build_envs[compile_group_idx].StaticObject( - target=str(obj_path.with_suffix(".o")), - source=str(src_path.resolve()), - ) - ) return objects def build_library( - default_env: Environment, lib_config: dict, project_src_dir: pathlib.Path, prepend_dir: pathlib.Path | None=None, debug_allowed=True + default_env: Environment, lib_config: dict, project_src_dir: pathlib.Path, framework_dir: pathlib.Path, framework_obj_dir: pathlib.Path ): lib_name = lib_config["nameOnDisk"] lib_path = lib_config["paths"]["build"] - if prepend_dir: - lib_path = prepend_dir / lib_path lib_objects = compile_source_files( - lib_config, default_env, project_src_dir, prepend_dir, debug_allowed + lib_config, default_env, project_src_dir, framework_dir, framework_obj_dir ) #print(f"Created build rule for " + str(pathlib.Path("$BUILD_DIR") / lib_path / lib_name)) @@ -153,6 +156,17 @@ def extract_flags(target_json: dict) -> dict[str, list[str]]: "CXXFLAGS": default_flags.get("CXX"), } +def find_included_files(environment: Environment) -> set[str]: + """ + Process a list of flags produced by extract_flags() to find files manually included by '-include' + """ + result = set() + for flag_var in ["CFLAGS", "CXXFLAGS", "CCFLAGS"]: + language_flags = environment.get(flag_var) + for index in range(0, len(language_flags)): + if language_flags[index] == "-include" and index < len(language_flags) - 1: + result.add(language_flags[index + 1]) + return result def extract_includes(target_json: dict) -> dict[str, list[str]]: """ From b0fec18ec1077f7c09ff3133b9541546831e7065 Mon Sep 17 00:00:00 2001 From: Jamie Smith Date: Sat, 6 Sep 2025 12:21:09 -0700 Subject: [PATCH 08/15] Hyphen to underscore --- .../python/mbed_platformio/{build-mbed-ce.py => build_mbed_ce.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tools/python/mbed_platformio/{build-mbed-ce.py => build_mbed_ce.py} (100%) diff --git a/tools/python/mbed_platformio/build-mbed-ce.py b/tools/python/mbed_platformio/build_mbed_ce.py similarity index 100% rename from tools/python/mbed_platformio/build-mbed-ce.py rename to tools/python/mbed_platformio/build_mbed_ce.py From 38ed9923ccd2061d030cd09bf95b83069bb7baf5 Mon Sep 17 00:00:00 2001 From: Jamie Smith Date: Sat, 6 Sep 2025 12:31:35 -0700 Subject: [PATCH 09/15] Remove debug comment --- tools/python/mbed_platformio/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/python/mbed_platformio/CMakeLists.txt b/tools/python/mbed_platformio/CMakeLists.txt index 73b21616cda..f3bceb21157 100644 --- a/tools/python/mbed_platformio/CMakeLists.txt +++ b/tools/python/mbed_platformio/CMakeLists.txt @@ -6,7 +6,6 @@ cmake_minimum_required(VERSION 3.19) cmake_policy(VERSION 3.19...3.22) set(MBED_APP_JSON_PATH ${PLATFORMIO_PROJECT_PATH}/mbed_app.json5) -message("MBED_APP_JSON_PATH = ${MBED_APP_JSON_PATH}") include(${PLATFORMIO_MBED_OS_PATH}/tools/cmake/mbed_toolchain_setup.cmake) project(PlatformIOMbedProject From eac1143e2f002b64c9da7627a571e09f48337de0 Mon Sep 17 00:00:00 2001 From: Jamie Smith Date: Sat, 6 Sep 2025 12:37:16 -0700 Subject: [PATCH 10/15] Add licenses --- tools/pyproject.toml | 2 ++ tools/python/mbed_platformio/CMakeLists.txt | 2 +- tools/python/mbed_platformio/__init__.py | 4 ++++ tools/python/mbed_platformio/build_mbed_ce.py | 3 +++ tools/python/mbed_platformio/cmake_to_scons_converter.py | 3 +++ tools/python/mbed_platformio/dummy_source_file.S | 4 +++- tools/python/mbed_platformio/dummy_source_file.c | 4 +++- tools/python/mbed_platformio/dummy_source_file.cpp | 4 +++- tools/python/mbed_platformio/pio_variants.py | 9 +++++++-- 9 files changed, 29 insertions(+), 6 deletions(-) diff --git a/tools/pyproject.toml b/tools/pyproject.toml index 4e06336e688..f96f0d00b29 100644 --- a/tools/pyproject.toml +++ b/tools/pyproject.toml @@ -72,6 +72,8 @@ unit-tests = [ "lxml" ] +# Install this optional dependency group to get IDE completion for packages used by the mbed_platformio package. +# Note that this package is actually run in platformio's venv, not the Mbed venv, so there is no other need to install these. platformio = [ "SCons", "platformio" diff --git a/tools/python/mbed_platformio/CMakeLists.txt b/tools/python/mbed_platformio/CMakeLists.txt index f3bceb21157..4e9ca6e7b2d 100644 --- a/tools/python/mbed_platformio/CMakeLists.txt +++ b/tools/python/mbed_platformio/CMakeLists.txt @@ -14,7 +14,7 @@ include(mbed_project_setup) add_subdirectory(${PLATFORMIO_MBED_OS_PATH} mbed-os) -# Dummy executable. Not built, but needed to get the linker flags via the CMake file API +# Dummy executable. Not built, but needed to get the compile and linker flags via the CMake file API add_executable(PIODummyExecutable dummy_source_file.c dummy_source_file.cpp diff --git a/tools/python/mbed_platformio/__init__.py b/tools/python/mbed_platformio/__init__.py index e69de29bb2d..8546dded510 100644 --- a/tools/python/mbed_platformio/__init__.py +++ b/tools/python/mbed_platformio/__init__.py @@ -0,0 +1,4 @@ +""" +Copyright (c) 2025 Jamie Smith +SPDX-License-Identifier: Apache-2.0 +""" \ No newline at end of file diff --git a/tools/python/mbed_platformio/build_mbed_ce.py b/tools/python/mbed_platformio/build_mbed_ce.py index e45809033e5..1db7a3356ee 100644 --- a/tools/python/mbed_platformio/build_mbed_ce.py +++ b/tools/python/mbed_platformio/build_mbed_ce.py @@ -3,6 +3,9 @@ This script acts as an SCons buildfile which gets configuration from PlatformIO, configures Mbed (if needed), and returns information about the configuration to the PIO build system. + +Copyright (c) 2025 Jamie Smith +SPDX-License-Identifier: Apache-2.0 """ from __future__ import annotations diff --git a/tools/python/mbed_platformio/cmake_to_scons_converter.py b/tools/python/mbed_platformio/cmake_to_scons_converter.py index 37e4c75501f..272c0cc8336 100644 --- a/tools/python/mbed_platformio/cmake_to_scons_converter.py +++ b/tools/python/mbed_platformio/cmake_to_scons_converter.py @@ -1,5 +1,8 @@ """ Functions for converting build system information from the CMake File API into SCons build targets. + +Copyright (c) 2025 Jamie Smith +SPDX-License-Identifier: Apache-2.0 """ from __future__ import annotations diff --git a/tools/python/mbed_platformio/dummy_source_file.S b/tools/python/mbed_platformio/dummy_source_file.S index cfd852831b2..4496a7c23a2 100644 --- a/tools/python/mbed_platformio/dummy_source_file.S +++ b/tools/python/mbed_platformio/dummy_source_file.S @@ -1 +1,3 @@ -// Empty C++ source file so that we can scan the ASM flags via the CMake API \ No newline at end of file +// Empty ASM source file so that we can scan the ASM flags via the CMake API +// Copyright (c) 2025 Jamie Smith +// SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/tools/python/mbed_platformio/dummy_source_file.c b/tools/python/mbed_platformio/dummy_source_file.c index c0b97ea8116..ab3ee9857ab 100644 --- a/tools/python/mbed_platformio/dummy_source_file.c +++ b/tools/python/mbed_platformio/dummy_source_file.c @@ -1 +1,3 @@ -// Empty C++ source file so that we can scan the C flags via the CMake API \ No newline at end of file +// Empty C++ source file so that we can scan the C++ flags via the CMake API +// Copyright (c) 2025 Jamie Smith +// SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/tools/python/mbed_platformio/dummy_source_file.cpp b/tools/python/mbed_platformio/dummy_source_file.cpp index a3b324ab247..ab3ee9857ab 100644 --- a/tools/python/mbed_platformio/dummy_source_file.cpp +++ b/tools/python/mbed_platformio/dummy_source_file.cpp @@ -1 +1,3 @@ -// Empty C++ source file so that we can scan the C++ flags via the CMake API \ No newline at end of file +// Empty C++ source file so that we can scan the C++ flags via the CMake API +// Copyright (c) 2025 Jamie Smith +// SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/tools/python/mbed_platformio/pio_variants.py b/tools/python/mbed_platformio/pio_variants.py index afcb6615ab2..76aa4b16c27 100644 --- a/tools/python/mbed_platformio/pio_variants.py +++ b/tools/python/mbed_platformio/pio_variants.py @@ -1,6 +1,11 @@ +""" +Maps PIO variant name to Mbed target name, in the situation where the Mbed target name is different +from the uppercased version of the variant name + +Copyright (c) 2025 Jamie Smith +SPDX-License-Identifier: Apache-2.0 +""" -# Maps PIO variant name to Mbed target name, in the situation where the Mbed target name is different -# from the uppercased version of the variant name PIO_VARIANT_TO_MBED_TARGET = { "seeedArchPro": "ARCH_PRO", "seeedArchMax": "ARCH_MAX", From abc47eb8c05471f3dd1f24488cc75cf8655bbbb7 Mon Sep 17 00:00:00 2001 From: Jamie Smith Date: Sat, 6 Sep 2025 16:29:49 -0700 Subject: [PATCH 11/15] Missed package.json --- package.json | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 package.json diff --git a/package.json b/package.json new file mode 100644 index 00000000000..5094caad2d0 --- /dev/null +++ b/package.json @@ -0,0 +1,17 @@ +{ + "name": "framework-mbed-ce", + "version": "7.0", + "title": "Mbed OS Community Edition", + "description": "Mbed CE is a platform operating system designed for the internet of things", + "keywords": [ + "framework", + "os", + "arm", + "hal" + ], + "homepage": "http://mbed-ce.dev", + "repository": { + "type": "git", + "url": "https://github.com/mbed-ce/mbed-os" + } +} From c716bd589123071acc5e9ce244b04bbb9fb8cde5 Mon Sep 17 00:00:00 2001 From: Jamie Smith Date: Sat, 6 Sep 2025 16:55:58 -0700 Subject: [PATCH 12/15] Final variable name --- tools/python/mbed_platformio/build_mbed_ce.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/python/mbed_platformio/build_mbed_ce.py b/tools/python/mbed_platformio/build_mbed_ce.py index 1db7a3356ee..5d9f6685769 100644 --- a/tools/python/mbed_platformio/build_mbed_ce.py +++ b/tools/python/mbed_platformio/build_mbed_ce.py @@ -289,7 +289,7 @@ def get_app_defines(app_config: dict): no_whole_archive = env.Command('$PROJECT_DIR/-Wl,--no-whole-archive', [], '') # Add the Mbed OS archive and the link flags at the end of the linker command line after all the app source files -env.Append(PIOAPPENDBUILDFILES=[whole_archive, "$BUILD_DIR\\mbed-os\\libmbed-os.a", no_whole_archive]) +env.Append(PIO_FRAMEWORK_APPEND_OBJS=[whole_archive, "$BUILD_DIR\\mbed-os\\libmbed-os.a", no_whole_archive]) # # Main environment configuration From 71f03d2892e46778147d322c7a84d5ecbdcde99f Mon Sep 17 00:00:00 2001 From: Jamie Smith Date: Sat, 6 Sep 2025 16:56:36 -0700 Subject: [PATCH 13/15] Final-er name --- tools/python/mbed_platformio/build_mbed_ce.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/python/mbed_platformio/build_mbed_ce.py b/tools/python/mbed_platformio/build_mbed_ce.py index 5d9f6685769..ba2560f51d6 100644 --- a/tools/python/mbed_platformio/build_mbed_ce.py +++ b/tools/python/mbed_platformio/build_mbed_ce.py @@ -289,7 +289,7 @@ def get_app_defines(app_config: dict): no_whole_archive = env.Command('$PROJECT_DIR/-Wl,--no-whole-archive', [], '') # Add the Mbed OS archive and the link flags at the end of the linker command line after all the app source files -env.Append(PIO_FRAMEWORK_APPEND_OBJS=[whole_archive, "$BUILD_DIR\\mbed-os\\libmbed-os.a", no_whole_archive]) +env.Append(PIO_EXTRA_APP_OBJS=[whole_archive, "$BUILD_DIR\\mbed-os\\libmbed-os.a", no_whole_archive]) # # Main environment configuration From d8d1f39c5c68da38e47c339e7af164c02bcf2172 Mon Sep 17 00:00:00 2001 From: Jamie Smith Date: Mon, 8 Sep 2025 08:23:07 -0700 Subject: [PATCH 14/15] Avoid need for one PIO Core change --- tools/python/mbed_platformio/build_mbed_ce.py | 38 ++++++++++--------- .../cmake_to_scons_converter.py | 10 ++--- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/tools/python/mbed_platformio/build_mbed_ce.py b/tools/python/mbed_platformio/build_mbed_ce.py index ba2560f51d6..27233bf112d 100644 --- a/tools/python/mbed_platformio/build_mbed_ce.py +++ b/tools/python/mbed_platformio/build_mbed_ce.py @@ -235,6 +235,8 @@ def build_components( def get_app_defines(app_config: dict): return extract_defines(app_config["compileGroups"][0]) +## CMake configuration ------------------------------------------------------------------------------------------------- + project_codemodel = get_cmake_code_model( [ "-S", @@ -266,45 +268,47 @@ def get_app_defines(app_config: dict): [], ) +## Convert targets & flags from CMake to SCons ------------------------------------------------------------------------- + build_components(env, framework_components_map, PROJECT_DIR) mbed_os_lib_target_json = target_configs.get("mbed-os", {}) app_target_json = target_configs.get("PIODummyExecutable", {}) project_defines = get_app_defines(app_target_json) project_flags = extract_flags(app_target_json) -link_args = extract_link_args(app_target_json) app_includes = extract_includes(app_target_json) +## Linker flags -------------------------------------------------------------------------------------------------------- + +# Link the main Mbed OS library using -Wl,--whole-archive. This is needed for the resolution of weak symbols +# within this archive. +link_args = ["-Wl,--whole-archive", "$BUILD_DIR\\mbed-os\\libmbed-os.a", "-Wl,--no-whole-archive"] +env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", "$BUILD_DIR\\mbed-os\\libmbed-os.a") + +# Get other linker flags from Mbed. We want these to appear after the application objects and Mbed libraries +# because they contain the C/C++ library link flags. +link_args.extend(extract_link_args(app_target_json)) + # The CMake build system adds a flag in mbed_set_post_build() to output a map file. # We need to do that here. map_file = BUILD_DIR / 'firmware.map' -link_args["LINKFLAGS"].append(f"-Wl,-Map={str(map_file)}") +link_args.append(f"-Wl,-Map={str(map_file)}") + +## Build environment configuration ------------------------------------------------------------------------------------- -# Link the main Mbed OS library using -Wl,--whole-archive. This is needed for the resolution of weak symbols -# within this archive. I found this workaround here on the mailing list: -# https://scons-users.scons.narkive.com/noJni4Ut/workaround-support-for-wl-whole-archive -# However, for some reason SCons need to think that the flag is "in the source dir" or it appends a path -# to the build dir before it. So, I added the $PROJECT_DIR part. -whole_archive = env.Command('$PROJECT_DIR/-Wl,--whole-archive', [], '') -no_whole_archive = env.Command('$PROJECT_DIR/-Wl,--no-whole-archive', [], '') - -# Add the Mbed OS archive and the link flags at the end of the linker command line after all the app source files -env.Append(PIO_EXTRA_APP_OBJS=[whole_archive, "$BUILD_DIR\\mbed-os\\libmbed-os.a", no_whole_archive]) - -# -# Main environment configuration -# -project_flags.update(link_args) env.MergeFlags(project_flags) env.Prepend( CPPPATH=app_includes["plain_includes"], CPPDEFINES=project_defines, ) +env.Append(_LIBFLAGS=link_args) # Set up a dependency between all application source files and mbed-target-config.h. # This ensures that the app will be recompiled if the header changes. env.Append(PIO_EXTRA_APP_SOURCE_DEPS=find_included_files(env)) +## Linker script ------------------------------------------------------------------------------------------------------- + # Run Ninja to produce the linker script. # Note that this seems to execute CMake, causing the code model query to be re-done. # So, we have to do this after we are done using the results of said query. diff --git a/tools/python/mbed_platformio/cmake_to_scons_converter.py b/tools/python/mbed_platformio/cmake_to_scons_converter.py index 272c0cc8336..f733c3791a0 100644 --- a/tools/python/mbed_platformio/cmake_to_scons_converter.py +++ b/tools/python/mbed_platformio/cmake_to_scons_converter.py @@ -187,12 +187,12 @@ def extract_includes(target_json: dict) -> dict[str, list[str]]: return {"plain_includes": plain_includes, "sys_includes": sys_includes} -def extract_link_args(target_json: dict) -> dict[str, list[str]]: +def extract_link_args(target_json: dict) -> list[str]: """ - Extract the linker flags from a CMake target and return an SCons-style dict + Extract the linker flags from a CMake target """ - link_args = {"LINKFLAGS": []} + result = [] for f in target_json.get("link", {}).get("commandFragments", []): fragment = f.get("fragment", "").strip() @@ -201,6 +201,6 @@ def extract_link_args(target_json: dict) -> dict[str, list[str]]: continue args = click.parser.split_arg_string(fragment) if fragment_role == "flags": - link_args["LINKFLAGS"].extend(args) + result.extend(args) - return link_args \ No newline at end of file + return result \ No newline at end of file From 628c99f03d35723f514004153a9c14aec90b89f4 Mon Sep 17 00:00:00 2001 From: Jamie Smith Date: Mon, 8 Sep 2025 08:23:31 -0700 Subject: [PATCH 15/15] Declare prerelease version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5094caad2d0..300df4c30ee 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "framework-mbed-ce", - "version": "7.0", + "version": "6.99.0", "title": "Mbed OS Community Edition", "description": "Mbed CE is a platform operating system designed for the internet of things", "keywords": [