Update support for ESP-IDF according to v3.3 // Issue #224

Author: valeros

builder/frameworks/espidf.py

@@ -157,11 +157,21 @@ def build_component(path, component_config):
         CPPDEFINES=component_config.get("cpp_defines", []),
     )
 
-    return envsafe.BuildLibrary(
+    component = envsafe.BuildLibrary(
         join("$BUILD_DIR", "%s" % basename(path)), path,
-        src_filter=component_config.get("src_filter", "+<*> -<test*>")
+        src_filter=component_config.get("src_filter", "+<*> -<test*>"))
+
+    component_sections = env.Command(
+        "${SOURCE}.sections_info", component,
+        env.VerboseAction(
+            "xtensa-esp32-elf-objdump -h $SOURCE > $TARGET", "Generating $TARGET")
     )
 
+    # Section files are used in linker script generation process
+    env.Depends("$BUILD_DIR/esp32.project.ld", component_sections)
+
+    return component
+
 
 def get_sdk_configuration(config_path):
     if not isfile(config_path):
@@ -180,13 +190,13 @@ def get_sdk_configuration(config_path):
     return config
 
 
-def find_valid_config_file():
+def find_valid_example_file(filename):
     search_path = join(
-        platform.get_dir(), "examples", "*", "src", "sdkconfig.h")
+        platform.get_dir(), "examples", "*", "src", filename)
     files = glob(search_path)
     if not files:
         sys.stderr.write(
-            "Error: Could not find default \"sdkconfig.h\" file\n")
+            "Error: Could not find default %s file\n" % filename)
         env.Exit(1)
     return files[0]
 
@@ -302,6 +312,26 @@ def build_wpa_supplicant_lib():
         join(FRAMEWORK_DIR, "components", "wpa_supplicant"), config)
 
 
+def build_wifi_provisioning_lib():
+    src_filter = "-<*>"
+    for d in ["proto-c", "src"]:
+        src_filter += " +<%s>" % d
+
+    if not (is_set("CONFIG_BT_ENABLED", sdk_params) and is_set(
+            "CONFIG_BLUEDROID_ENABLED", sdk_params)):
+        src_filter += " -<src/scheme_ble.c>"
+
+    inc_dirs = ["src", "proto-c", "../protocomm/proto-c"]
+
+    config = {
+        "inc_dirs": inc_dirs,
+        "src_filter": src_filter
+    }
+
+    return build_component(
+        join(FRAMEWORK_DIR, "components", "wifi_provisioning"), config)
+
+
 def build_heap_lib(sdk_params):
     src_filter = "+<*> -<test*>"
     if is_set("CONFIG_HEAP_POISONING_DISABLED", sdk_params):
@@ -375,11 +405,68 @@ def build_espidf_bootloader():
         join("$BUILD_DIR", "bootloader.elf"),
         envsafe.CollectBuildFiles(
             join("$BUILD_DIR", "bootloader"),
-            join(FRAMEWORK_DIR, "components", "bootloader", "subproject", "main")
+            join(FRAMEWORK_DIR, "components", "bootloader", "subproject",
+                 "main")
         )
     )
 
 
+def generate_section_info(target, source, env):
+    with open(target[0].get_path(), "w") as fp:
+        fp.write("\n".join(s.get_path() + ".sections_info" for s in source))
+
+    return None
+
+
+def find_framework_service_files(search_path):
+    result = {}
+    result['lf_files'] = list()
+    result['kconfig_files'] = list()
+    result['kconfig_build_files'] = list()
+    for d in listdir(search_path):
+        for f in listdir(join(search_path, d)):
+            if f == "linker.lf":
+                result['lf_files'].append(join(search_path, d, f))
+            elif f == "Kconfig.projbuild":
+                result['kconfig_build_files'].append(join(search_path, d, f))
+            elif f == "Kconfig":
+                result['kconfig_files'].append(join(search_path, d, f))
+
+    result['lf_files'].append(
+        join(FRAMEWORK_DIR, "components", "esp32", "ld", "esp32_fragments.lf"))
+
+    return result
+
+
+def generate_project_ld_script(target, source, env):
+    project_files = find_framework_service_files(
+        join(FRAMEWORK_DIR, "components"))
+
+    args = {
+        "script": join(FRAMEWORK_DIR, "tools", "ldgen", "ldgen.py"),
+        "sections": source[0].get_path(),
+        "input": join(
+            FRAMEWORK_DIR, "components", "esp32", "ld", "esp32.project.ld.in"),
+        "sdk_header": join(env.subst("$PROJECTSRC_DIR"), "sdkconfig.h"),
+        "fragments": " ".join(
+            ["\"%s\"" % f for f in project_files.get("lf_files")]),
+        "output": target[0].get_path(),
+        "kconfig": join(FRAMEWORK_DIR, "Kconfig"),
+        "kconfigs_projbuild": "COMPONENT_KCONFIGS_PROJBUILD=%s" % "#".join(
+            ["%s" % f for f in project_files.get("kconfig_build_files")]),
+        "kconfig_files": "COMPONENT_KCONFIGS=%s" % "#".join(
+            ["%s" % f for f in project_files.get("kconfig_files")]),
+    }
+
+    cmd = ('"$PYTHONEXE" "{script}" --sections "{sections}" --input "{input}" '
+        '--config "{sdk_header}" --fragments {fragments} --output "{output}" '
+        '--kconfig "{kconfig}" --env "{kconfigs_projbuild}" '
+        '--env "{kconfig_files}" --env "IDF_CMAKE=n" '
+        '--env "IDF_TARGET=\\\"esp32\\\""').format(**args)
+
+    env.Execute(cmd)
+
+
 def build_arduino_framework():
     core = env.BoardConfig().get("build.core")
 
@@ -473,14 +560,18 @@ def configure_exceptions(sdk_params):
              "libcoap", "include", "coap"),
         join(FRAMEWORK_DIR, "components", "console"),
         join(FRAMEWORK_DIR, "components", "driver", "include"),
+        join(FRAMEWORK_DIR, "components", "efuse", "include"),
+        join(FRAMEWORK_DIR, "components", "efuse", "esp32", "include"),
         join(FRAMEWORK_DIR, "components", "esp-tls"),
         join(FRAMEWORK_DIR, "components", "esp_adc_cal", "include"),
         join(FRAMEWORK_DIR, "components", "esp_event", "include"),
         join(FRAMEWORK_DIR, "components", "esp_http_client", "include"),
         join(FRAMEWORK_DIR, "components", "esp_http_server", "include"),
+        join(FRAMEWORK_DIR, "components", "esp_https_server", "include"),
         join(FRAMEWORK_DIR, "components", "esp_https_ota", "include"),
         join(FRAMEWORK_DIR, "components", "esp_ringbuf", "include"),
         join(FRAMEWORK_DIR, "components", "esp32", "include"),
+        join(FRAMEWORK_DIR, "components", "espcoredump", "include"),
         join(FRAMEWORK_DIR, "components", "ethernet", "include"),
         join(FRAMEWORK_DIR, "components", "expat", "expat", "lib"),
         join(FRAMEWORK_DIR, "components", "expat", "port", "include"),
@@ -525,6 +616,8 @@ def configure_exceptions(sdk_params):
         join(FRAMEWORK_DIR, "components", "spiffs", "include"),
         join(FRAMEWORK_DIR, "components", "tcp_transport", "include"),
         join(FRAMEWORK_DIR, "components", "tcpip_adapter", "include"),
+        join(FRAMEWORK_DIR, "components", "unity", "include"),
+        join(FRAMEWORK_DIR, "components", "unity", "unity", "src"),
         join(FRAMEWORK_DIR, "components", "ulp", "include"),
         join(FRAMEWORK_DIR, "components", "vfs", "include"),
         join(FRAMEWORK_DIR, "components", "wear_levelling", "include"),
@@ -557,14 +650,17 @@ def configure_exceptions(sdk_params):
         if (d == "include"):
             env.Prepend(CPPPATH=[join(root, d)])
 
-
 env.Prepend(
     CFLAGS=["-Wno-old-style-declaration"],
 
     CPPDEFINES=[
         "WITH_POSIX",
+        "UNITY_INCLUDE_CONFIG_H",
         ("IDF_VER", '\\"%s\\"' %
-         platform.get_package_version("framework-espidf"))
+         platform.get_package_version("framework-espidf")),
+        ("PROJECT_VER", '\\"%s\\"' % "1.0.0"),
+        ("PROJECT_NAME", '\\"%s\\"' % basename(env.subst("$PROJECT_DIR")))
+
     ],
 
     CCFLAGS=[
@@ -582,8 +678,9 @@ def configure_exceptions(sdk_params):
 env.Append(
     LINKFLAGS=[
         "-u", "__cxa_guard_dummy",
+        "-u", "esp_app_desc",
         "-u", "ld_include_panic_highint_hdl",
-        "-T", "esp32.common.ld",
+        "-T", "esp32.project.ld",
         "-T", "esp32.rom.ld",
         "-T", "esp32.peripherals.ld",
         "-T", "esp32.rom.libgcc.ld",
@@ -607,35 +704,40 @@ def configure_exceptions(sdk_params):
 env.Replace(ASFLAGS=[])
 
 #
-# Handle missing sdkconfig.h
+# Handle missing sdkconfig files
 #
 
-sdk_config_file = join(env.subst("$PROJECTSRC_DIR"), "sdkconfig.h")
 
-if not isfile(sdk_config_file):
-    print("Warning! Cannot find \"sdkconfig.h\" file. "
-          "Default \"sdkconfig.h\" will be added to your project!")
-    copy(find_valid_config_file(), sdk_config_file)
-else:
-    is_new = False
-    with open(sdk_config_file) as fp:
-        for l in fp.readlines():
-            if "CONFIG_PTHREAD_STACK_MIN" in l:
-                is_new = True
-                break
-
-    if not is_new:
-        print("Warning! Detected an outdated \"sdkconfig.h\" file. "
-              "The old \"sdkconfig.h\" will be replaced by the new one.")
-
-        new_config = find_valid_config_file()
-        copy(
-            sdk_config_file,
-            join(env.subst("$PROJECTSRC_DIR"), "sdkconfig.h.bak")
-        )
-        copy(new_config, sdk_config_file)
+def process_project_configs(filename):
+    config = join(env.subst("$PROJECTSRC_DIR"), filename)
+    if not isfile(config):
+        print("Warning! Cannot find %s file. Default "
+              "file will be added to your project!" % filename)
+        copy(find_valid_example_file(filename), config)
+    else:
+        is_new = False
+        with open(config) as fp:
+            for l in fp.readlines():
+                if "CONFIG_APP_COMPILE_TIME_DATE" in l:
+                    is_new = True
+                    break
+
+        if not is_new:
+            print("Warning! Detected an outdated %s file. The old "
+                  "file will be replaced by the new one." % filename)
+
+            new_config = find_valid_example_file(filename)
+            copy(config, join(
+                env.subst("$PROJECTSRC_DIR"), "%s.bak" % filename))
+            copy(new_config, config)
+
 
-sdk_params = get_sdk_configuration(sdk_config_file)
+process_project_configs("sdkconfig")
+sdk_config = join(env.subst("$PROJECTSRC_DIR"), "sdkconfig")
+
+process_project_configs("sdkconfig.h")
+sdk_config_header = join(env.subst("$PROJECTSRC_DIR"), "sdkconfig.h")
+sdk_params = get_sdk_configuration(sdk_config_header)
 
 configure_exceptions(sdk_params)
 
@@ -689,35 +791,23 @@ def configure_exceptions(sdk_params):
 
 libs = []
 
-if ulp_lib:
-    if not is_ulp_enabled(sdk_params):
-        print("Warning! ULP is not properly configured. "
-              "Add the next configuration lines to your sdkconfig.h:")
-        print ("    #define CONFIG_ULP_COPROC_ENABLED 1")
-        print ("    #define CONFIG_ULP_COPROC_RESERVE_MEM 1024")
-
-    libs.append(ulp_lib)
-    env.Append(
-        CPPPATH=[join("$BUILD_DIR", "ulp_app")],
-        LIBPATH=[join("$BUILD_DIR", "ulp_app")],
-        LINKFLAGS=["-T", "ulp_main.ld"]
-    )
-
 ignore_dirs = (
     "bootloader",
     "esptool_py",
-    "espcoredump",
     "idf_test",
     "partition_table"
 )
 
 special_src_filter = {
     "app_trace": "+<*> -<test> -<sys_view> -<gcov>",
+    "asio": "-<*> +<asio/asio/src/asio.cpp>",
     "aws_iot": "-<*> +<port> +<aws-iot-device-sdk-embedded-C/src>",
     "esp32": "-<*> +<*.[sSc]*> +<hwcrypto>",
     "bootloader_support": "+<*> -<test> -<src/bootloader_init.c>",
     "soc": "+<*> -<test> -<esp32/test>",
-    "spi_flash": "+<*> -<test*> -<sim>"
+    "spi_flash": "+<*> -<test*> -<sim>",
+    "efuse": "+<*> -<test*>",
+    "unity": "-<*> +<unity/src> +<*.c>"
 }
 
 special_env = (
@@ -726,17 +816,17 @@ def configure_exceptions(sdk_params):
     "lwip",
     "protocomm",
     "libsodium",
-    "wpa_supplicant"
+    "wpa_supplicant",
+    "wifi_provisioning"
 )
 
 for d in listdir(join(FRAMEWORK_DIR, "components")):
     if d in special_src_filter or d in special_env or d in ignore_dirs:
         continue
     component_dir = join(FRAMEWORK_DIR, "components", d)
     if isdir(component_dir):
-        libs.append(
-            build_component(component_dir,
-                            extract_component_config(component_dir)))
+        libs.append(build_component(
+            component_dir, extract_component_config(component_dir)))
 
 
 for component, src_filter in special_src_filter.items():
@@ -751,9 +841,35 @@ def configure_exceptions(sdk_params):
     build_heap_lib(sdk_params),
     build_rtos_lib(),
     build_libsodium_lib(),
-    build_wpa_supplicant_lib()
+    build_wpa_supplicant_lib(),
+    build_wifi_provisioning_lib()
 ))
 
+project_sections = env.Command(
+    join("$BUILD_DIR", "ldgen.section_infos"), libs, generate_section_info)
+
+project_linker_script = env.Command(
+    join("$BUILD_DIR", "esp32.project.ld"),
+    project_sections, generate_project_ld_script,
+    ENV={"PYTHONPATH": join(FRAMEWORK_DIR, "platformio", "package_deps",
+                            "py%d" % sys.version_info.major)})
+
+env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", project_linker_script)
+
+if ulp_lib:
+    if not is_ulp_enabled(sdk_params):
+        print("Warning! ULP is not properly configured."
+              "Add next configuration lines to your sdkconfig.h:")
+        print ("    #define CONFIG_ULP_COPROC_ENABLED 1")
+        print ("    #define CONFIG_ULP_COPROC_RESERVE_MEM 1024")
+
+    libs.append(ulp_lib)
+    env.Append(
+        CPPPATH=[join("$BUILD_DIR", "ulp_app")],
+        LIBPATH=[join("$BUILD_DIR", "ulp_app")],
+        LINKFLAGS=["-T", "ulp_main.ld"]
+    )
+
 #
 # Process Arduino core
 #