Refactor processing of IDF linker fragments

Resolve platformio#760

Author: valeros

builder/frameworks/espidf.py

@@ -429,57 +429,92 @@ def get_sdk_configuration():
         return {}
 
 
-def find_framework_service_files(search_path, sdk_config):
-    result = {}
-    result["lf_files"] = list()
-    result["kconfig_files"] = list()
-    result["kconfig_build_files"] = list()
-    for d in os.listdir(search_path):
-        path = os.path.join(search_path, d)
-        if not os.path.isdir(path):
-            continue
-        for f in os.listdir(path):
-            # Skip hardware specific files as they will be added later
-            if f == "linker.lf" and not os.path.basename(path).startswith(
-                ("esp32", "riscv")
+def load_component_paths(framework_components_dir, ignored_component_prefixes=None):
+    def _scan_components_from_framework():
+        result = []
+        for component in os.listdir(framework_components_dir):
+            component_path = os.path.join(framework_components_dir, component)
+            if component.startswith(ignored_component_prefixes) or not os.path.isdir(
+                component_path
             ):
-                result["lf_files"].append(os.path.join(path, f))
-            elif f == "Kconfig.projbuild":
-                result["kconfig_build_files"].append(os.path.join(path, f))
-            elif f == "Kconfig":
-                result["kconfig_files"].append(os.path.join(path, f))
+                continue
+            result.append(component_path)
+
+        return result
+
+    # First of all, try to load the list of used components from the project description
+    components = []
+    ignored_component_prefixes = ignored_component_prefixes or []
+    project_description_file = os.path.join(BUILD_DIR, "project_description.json")
+    if os.path.isfile(project_description_file):
+        with open(project_description_file) as fp:
+            try:
+                data = json.load(fp)
+                for path in data.get("build_component_paths", []):
+                    if not os.path.basename(path).startswith(
+                        ignored_component_prefixes
+                    ):
+                        components.append(path)
+            except:
+                print(
+                    "Warning: Could not find load components from project description!\n"
+                )
+
+    return components or _scan_components_from_framework()
+
+
+def extract_linker_script_fragments(framework_components_dir, sdk_config):
+    # Hardware-specific components are excluded from search and added manually below
+    project_components = load_component_paths(
+        framework_components_dir, ignored_component_prefixes=("esp32", "riscv")
+    )
+
+    result = []
+    for component_path in project_components:
+        linker_fragment = os.path.join(component_path, "linker.lf")
+        if os.path.isfile(linker_fragment):
+            result.append(linker_fragment)
+
+    if not result:
+        sys.stderr.write("Error: Failed to extract paths to linker script fragments\n")
+        env.Exit(1)
 
     if mcu == "esp32c3":
-        result["lf_files"].append(
-            os.path.join(FRAMEWORK_DIR, "components", "riscv", "linker.lf")
-        )
+        result.append(os.path.join(framework_components_dir, "riscv", "linker.lf"))
 
-    result["lf_files"].extend(
+    result.extend(
         [
             os.path.join(
-                FRAMEWORK_DIR,
-                "components",
+                framework_components_dir,
                 idf_variant,
                 "ld",
                 "%s_fragments.lf" % idf_variant,
             ),
             os.path.join(
-                FRAMEWORK_DIR,
-                "components",
+                framework_components_dir,
                 idf_variant,
                 "linker.lf",
             ),
-            os.path.join(FRAMEWORK_DIR, "components", "newlib", "newlib.lf"),
+            os.path.join(framework_components_dir, "newlib", "newlib.lf"),
         ]
     )
 
     if sdk_config.get("SPIRAM_CACHE_WORKAROUND", False):
-        result["lf_files"].append(
+        result.append(
             os.path.join(
-                FRAMEWORK_DIR, "components", "newlib", "esp32-spiram-rom-functions-c.lf"
+                framework_components_dir, "newlib", "esp32-spiram-rom-functions-c.lf"
             )
         )
 
+    if board.get("build.esp-idf.extra_lf_files", ""):
+        result.extend(
+            [
+                lf if os.path.isabs(lf) else os.path.join(PROJECT_DIR, lf)
+                for lf in board.get("build.esp-idf.extra_lf_files").splitlines()
+                if lf.strip()
+            ]
+        )
+
     return result
 
 
@@ -510,7 +545,7 @@ def create_custom_libraries_list(ldgen_libraries_file, ignore_targets):
 
 def generate_project_ld_script(sdk_config, ignore_targets=None):
     ignore_targets = ignore_targets or []
-    project_files = find_framework_service_files(
+    linker_script_fragments = extract_linker_script_fragments(
         os.path.join(FRAMEWORK_DIR, "components"), sdk_config
     )
 
@@ -523,7 +558,9 @@ def generate_project_ld_script(sdk_config, ignore_targets=None):
     args = {
         "script": os.path.join(FRAMEWORK_DIR, "tools", "ldgen", "ldgen.py"),
         "config": SDKCONFIG_PATH,
-        "fragments": " ".join(['"%s"' % f for f in project_files.get("lf_files")]),
+        "fragments": " ".join(
+            ['"%s"' % fs.to_unix_path(f) for f in linker_script_fragments]
+        ),
         "kconfig": os.path.join(FRAMEWORK_DIR, "Kconfig"),
         "env_file": os.path.join("$BUILD_DIR", "config.env"),
         "libraries_list": libraries_list,
@@ -589,7 +626,9 @@ def prepare_build_envs(config, default_env, debug_allowed=True):
     return build_envs
 
 
-def compile_source_files(config, default_env, project_src_dir, prepend_dir=None, debug_allowed=True):
+def compile_source_files(
+    config, default_env, project_src_dir, prepend_dir=None, debug_allowed=True
+):
     build_envs = prepare_build_envs(config, default_env, debug_allowed)
     objects = []
     components_dir = fs.to_unix_path(os.path.join(FRAMEWORK_DIR, "components"))