Migration to ESP32 IDF 4.4 (#2246)

Author: josesimoes

.devcontainer/Dockerfile.All

@@ -1 +1 @@
-FROM ghcr.io/nanoframework/dev-container-all:v2.5
+FROM ghcr.io/nanoframework/dev-container-all:v2.6

.devcontainer/Dockerfile.ESP32

@@ -1 +1 @@
-FROM ghcr.io/nanoframework/dev-container-esp32:v2.4
+FROM ghcr.io/nanoframework/dev-container-esp32:v2.5

.devcontainer/sources/Dockerfile.All

@@ -74,7 +74,7 @@ RUN git clone --branch STABLE-2_0_3_RELEASE https://git.savannah.nongnu.org/git/
 ENV GIT_SSL_NO_VERIFY=0
 
 # Clone ESP-IDF
-RUN git clone --branch v4.3.2 https://github.com/espressif/esp-idf --depth 1 --recursive ./sources/esp-idf
+RUN git clone --branch v4.4 https://github.com/espressif/esp-idf --depth 1 --recursive ./sources/esp-idf
 
 # Clone what is needed for TI
 RUN git clone --branch 4.10.00.07 https://github.com/nanoframework/SimpleLink_CC32xx_SDK.git --depth 1 ./sources/SimpleLinkCC32 \
@@ -104,7 +104,7 @@ RUN ln -fs /usr/bin/python3 /usr/bin/python \
 ENV IDF_PATH=/sources/esp-idf
 RUN python -m pip install -r $IDF_PATH/requirements.txt
 RUN $IDF_PATH/install.sh
-ENV PATH=$PATH:/$IDF_PATH/components/esptool_py/esptool:/$IDF_PATH/components/espcoredump:/$IDF_PATH/components/partition_table/:/$IDF_PATH/tools/:$IDF_PATH/components/app_update:/root/.espressif/tools/xtensa-esp32-elf/esp-2021r2-8.4.0/xtensa-esp32-elf/bin:/root/.espressif/tools/xtensa-esp32s2-elf/esp-2021r2-8.4.0/xtensa-esp32s2-elf/bin:/root/.espressif/tools/riscv32-esp-elf/esp-2021r2-8.4.0/riscv32-esp-elf/bin
+ENV PATH=$PATH:/$IDF_PATH/components/esptool_py/esptool:/$IDF_PATH/components/espcoredump:/$IDF_PATH/components/partition_table/:/$IDF_PATH/tools/:$IDF_PATH/components/app_update:/root/.espressif/tools/xtensa-esp32-elf/esp-2021r2-patch2-8.4.0/xtensa-esp32-elf/bin:/root/.espressif/tools/xtensa-esp32s2-elf/esp-2021r2-patch2-8.4.0/xtensa-esp32s2-elf/bin:/root/.espressif/tools/riscv32-esp-elf/esp-2021r2-patch2-8.4.0/riscv32-esp-elf/bin
 
 # Clean up downloaded files
 RUN apt-get autoremove -y \

.devcontainer/sources/Dockerfile.ESP32

@@ -53,7 +53,7 @@ RUN git clone --branch V10.4.1-kernel-only https://github.com/FreeRTOS/FreeRTOS-
     && git clone --branch nf-build https://github.com/nanoframework/spiffs.git --depth 1 ./sources/spiffs
 
 # Clone ESP-IDF
-RUN git clone --branch v4.3.2 https://github.com/espressif/esp-idf --depth 1 --recursive ./sources/esp-idf
+RUN git clone --branch v4.4 https://github.com/espressif/esp-idf --depth 1 --recursive ./sources/esp-idf
 
 # Copy from our other container
 COPY --from=downloader /tmp/dc-extracted/gcc /usr/local/bin/gcc 
@@ -68,7 +68,7 @@ RUN ln -fs /usr/bin/python3 /usr/bin/python \
 ENV IDF_PATH=/sources/esp-idf
 RUN python -m pip install -r $IDF_PATH/requirements.txt
 RUN $IDF_PATH/install.sh
-ENV PATH=$PATH:/$IDF_PATH/components/esptool_py/esptool:/$IDF_PATH/components/espcoredump:/$IDF_PATH/components/partition_table/:/$IDF_PATH/tools/:$IDF_PATH/components/app_update:/root/.espressif/tools/xtensa-esp32-elf/esp-2021r2-8.4.0/xtensa-esp32-elf/bin:/root/.espressif/tools/xtensa-esp32s2-elf/esp-2021r2-8.4.0/xtensa-esp32s2-elf/bin:/root/.espressif/tools/riscv32-esp-elf/esp-2021r2-8.4.0/riscv32-esp-elf/bin
+ENV PATH=$PATH:/$IDF_PATH/components/esptool_py/esptool:/$IDF_PATH/components/espcoredump:/$IDF_PATH/components/partition_table/:/$IDF_PATH/tools/:$IDF_PATH/components/app_update:/root/.espressif/tools/xtensa-esp32-elf/esp-2021r2-patch2-8.4.0/xtensa-esp32-elf/bin:/root/.espressif/tools/xtensa-esp32s2-elf/esp-2021r2-patch2-8.4.0/xtensa-esp32s2-elf/bin:/root/.espressif/tools/riscv32-esp-elf/esp-2021r2-patch2-8.4.0/riscv32-esp-elf/bin
 
 # Clean up downloaded files
 RUN apt-get autoremove -y \

.github/workflows/all.yaml

@@ -2,7 +2,7 @@ name: Build and push image for all RTOS
 
 env:
   GCR_IMAGE: ghcr.io/nanoframework/dev-container-all
-  GCR_VERSION: v2.5
+  GCR_VERSION: v2.6
   GCR_FILE: .devcontainer/sources/Dockerfile.All
 
 # Controls when the action will run. 

.github/workflows/esp32.yml

@@ -2,7 +2,7 @@ name: Build and push ESP32 image
 
 env:
   GCR_IMAGE: ghcr.io/nanoframework/dev-container-esp32
-  GCR_VERSION: v2.4
+  GCR_VERSION: v2.5
   GCR_FILE: .devcontainer/sources/Dockerfile.ESP32
 
 # Controls when the action will run. 

CMake/Modules/FindESP32_IDF.cmake

@@ -16,7 +16,9 @@ list(APPEND ESP32_IDF_INCLUDE_DIRS ${esp32_idf_SOURCE_DIR}/components/xtensa/${T
 list(APPEND ESP32_IDF_INCLUDE_DIRS ${esp32_idf_SOURCE_DIR}/components/driver/${TARGET_SERIES_SHORT}/include/driver)
 list(APPEND ESP32_IDF_INCLUDE_DIRS ${esp32_idf_SOURCE_DIR}/components/hal/${TARGET_SERIES_SHORT}/include)
 list(APPEND ESP32_IDF_INCLUDE_DIRS ${esp32_idf_SOURCE_DIR}/components/soc/${TARGET_SERIES_SHORT}/include)
-
+list(APPEND ESP32_IDF_INCLUDE_DIRS ${esp32_idf_SOURCE_DIR}/components/soc/${TARGET_SERIES_SHORT}/include/soc)
+list(APPEND ESP32_IDF_INCLUDE_DIRS ${esp32_idf_SOURCE_DIR}/components/esp_hw_support/include/soc)
+                                                           
 list(APPEND ESP32_IDF_INCLUDE_DIRS ${esp32_idf_SOURCE_DIR}/components/driver/include)
 list(APPEND ESP32_IDF_INCLUDE_DIRS ${esp32_idf_SOURCE_DIR}/components/driver/include/driver)
 list(APPEND ESP32_IDF_INCLUDE_DIRS ${esp32_idf_SOURCE_DIR}/components/esp_common/include)
@@ -72,6 +74,8 @@ include(${TARGET_SERIES}_GCC_options)
 ############
 
 list(APPEND ESP32_IDF_INCLUDE_DIRS ${esp32_idf_SOURCE_DIR}/components/freertos/include)
+list(APPEND ESP32_IDF_INCLUDE_DIRS ${esp32_idf_SOURCE_DIR}/components/freertos/include/esp_additions)
+list(APPEND ESP32_IDF_INCLUDE_DIRS ${esp32_idf_SOURCE_DIR}/components/freertos/include/esp_additions/freertos)
 list(APPEND ESP32_IDF_INCLUDE_DIRS ${esp32_idf_SOURCE_DIR}/components/freertos/port/xtensa/include)
 list(APPEND ESP32_IDF_INCLUDE_DIRS ${esp32_idf_SOURCE_DIR}/components/freertos/port/xtensa/include/freertos)
 

CMake/Modules/FindmbedTLS.cmake

@@ -40,7 +40,6 @@ set(src_crypto
     asn1parse.c
     asn1write.c
     base64.c
-    bignum.c
     blowfish.c
     camellia.c
     ccm.c
@@ -90,20 +89,21 @@ set(src_crypto
     version.c
     version_features.c
     xtea.c
-    
+
 )
 
 if(NOT RTOS_ESP32_CHECK)
     # platform implementation of hardware random provider
     list(APPEND src_crypto mbedtls_entropy_hardware_pool.c)
+    list(APPEND src_crypto bignum.c)
 endif()
 
 foreach(SRC_FILE ${src_crypto})
 
     set(MBEDTLS_SRC_FILE SRC_FILE -NOTFOUND)
 
     find_file(MBEDTLS_SRC_FILE ${SRC_FILE}
-        PATHS 
+        PATHS
             ${mbedtls_SOURCE_DIR}/library
 
             ${BASE_PATH_FOR_CLASS_LIBRARIES_MODULES}/
@@ -119,6 +119,10 @@ foreach(SRC_FILE ${src_crypto})
 
 endforeach()
 
+if(RTOS_ESP32_CHECK)
+    list(APPEND mbedTLS_SOURCES ${esp32_idf_SOURCE_DIR}/components/mbedtls/mbedtls/library/bignum.c)
+endif()
+
 # unset this warning as error required for this source file
 SET_SOURCE_FILES_PROPERTIES( ${mbedtls_SOURCE_DIR}/library/hmac_drbg.c PROPERTIES COMPILE_FLAGS -Wno-maybe-uninitialized)
 SET_SOURCE_FILES_PROPERTIES( ${mbedtls_SOURCE_DIR}/library/x509_crt.c PROPERTIES COMPILE_FLAGS -Wno-maybe-uninitialized)
@@ -140,7 +144,7 @@ foreach(SRC_FILE ${src_x509})
     set(MBEDTLS_SRC_FILE SRC_FILE -NOTFOUND)
 
     find_file(MBEDTLS_SRC_FILE ${SRC_FILE}
-        PATHS 
+        PATHS
             ${mbedtls_SOURCE_DIR}/library
 
         CMAKE_FIND_ROOT_PATH_BOTH
@@ -172,7 +176,7 @@ foreach(SRC_FILE ${src_tls})
     set(MBEDTLS_SRC_FILE SRC_FILE -NOTFOUND)
 
     find_file(MBEDTLS_SRC_FILE ${SRC_FILE}
-        PATHS 
+        PATHS
             ${mbedtls_SOURCE_DIR}/library
 
         CMAKE_FIND_ROOT_PATH_BOTH
@@ -193,7 +197,6 @@ if(RTOS_ESP32_CHECK)
     set(src_platform_specific
 
         mbedtls_debug.c
-        
         esp_hardware.c
         esp_mem.c
         esp_timing.c
@@ -202,14 +205,45 @@ if(RTOS_ESP32_CHECK)
         esp_aes_common.c
         esp_aes.c
         sha.c
-        esp_bignum.c
-        bignum.c
         esp_sha1.c
         esp_sha256.c
         esp_sha512.c
         esp_md.c
     )
- 
+        
+    foreach(SRC_FILE ${src_platform_specific})
+
+    set(MBEDTLS_SRC_FILE SRC_FILE -NOTFOUND)
+
+    find_file(MBEDTLS_SRC_FILE ${SRC_FILE}
+        PATHS
+
+            ${esp32_idf_SOURCE_DIR}/components/mbedtls/port
+            ${esp32_idf_SOURCE_DIR}/components/mbedtls/port/sha/parallel_engine
+            ${esp32_idf_SOURCE_DIR}/components/mbedtls/port/sha/
+            ${esp32_idf_SOURCE_DIR}/components/mbedtls/port/aes
+            ${esp32_idf_SOURCE_DIR}/components/mbedtls/port/aes/block
+            
+            ${esp32_idf_SOURCE_DIR}/components/mbedtls/port/md
+
+            ${CMAKE_SOURCE_DIR}/targets/ESP32/_IDF
+
+        CMAKE_FIND_ROOT_PATH_BOTH
+    )
+
+    if (BUILD_VERBOSE)
+        message("${SRC_FILE} >> ${MBEDTLS_SRC_FILE}")
+    endif()
+
+    list(APPEND mbedTLS_SOURCES ${MBEDTLS_SRC_FILE})
+
+    endforeach()
+
+    if(CONFIG_MBEDTLS_HARDWARE_MPI)
+        list(APPEND mbedTLS_SOURCES ${esp32_idf_SOURCE_DIR}/components/mbedtls/port/esp_bignum.c)
+        list(APPEND src_platform_specific ${esp32_idf_SOURCE_DIR}/components/mbedtls/port/${TARGET_SERIES_SHORT}/bignum.c)
+    endif()
+
 else()
 
     # other platforms use the official sources
@@ -219,23 +253,13 @@ else()
         net_sockets.c
     )
 
-endif()
-
-foreach(SRC_FILE ${src_platform_specific})
+    foreach(SRC_FILE ${src_platform_specific})
 
     set(MBEDTLS_SRC_FILE SRC_FILE -NOTFOUND)
 
     find_file(MBEDTLS_SRC_FILE ${SRC_FILE}
         PATHS
 
-            ${esp32_idf_SOURCE_DIR}/components/mbedtls/port
-            ${esp32_idf_SOURCE_DIR}/components/mbedtls/port/sha/parallel_engine
-            ${esp32_idf_SOURCE_DIR}/components/mbedtls/port/sha/
-            ${esp32_idf_SOURCE_DIR}/components/mbedtls/port/aes
-            ${esp32_idf_SOURCE_DIR}/components/mbedtls/port/aes/block
-            ${esp32_idf_SOURCE_DIR}/components/mbedtls/port/${TARGET_SERIES_SHORT}
-            ${esp32_idf_SOURCE_DIR}/components/mbedtls/port/md
-
             ${mbedtls_SOURCE_DIR}/library
 
         CMAKE_FIND_ROOT_PATH_BOTH
@@ -247,7 +271,9 @@ foreach(SRC_FILE ${src_platform_specific})
 
     list(APPEND mbedTLS_SOURCES ${MBEDTLS_SRC_FILE})
 
-endforeach()
+    endforeach()
+
+endif()
 
 include(FindPackageHandleStandardArgs)
 

CMake/binutils.ESP32.cmake

@@ -165,10 +165,15 @@ macro(nf_add_platform_dependencies target)
             EXTRA_INCLUDES 
                 ${ESP32_IDF_INCLUDE_DIRS}
                 ${TARGET_ESP32_IDF_INCLUDES}
+                ${esp32_idf_SOURCE_DIR}/components/mbedtls/mbedtls/include
         )
 
         add_dependencies(${target}.elf nano::NF_Network)
 
+        # security provider is mbedTLS
+        if(USE_SECURITY_MBEDTLS_OPTION)
+            add_dependencies(NF_Network mbedtls)
+        endif()
     endif()
 
     # if(USE_FILESYSTEM_OPTION)
@@ -243,6 +248,14 @@ macro(nf_add_platform_sources target)
 
     endif()
 
+    if(USE_SECURITY_MBEDTLS_OPTION)
+        target_link_libraries(${target}.elf
+        mbedtls
+        )
+
+        add_dependencies(NF_Network mbedtls)
+    endif()
+
 endmacro()
 
 # macro to setup the build for a target
@@ -658,24 +671,6 @@ macro(nf_add_idf_as_library)
             PROPERTY COMPILE_DEFINITIONS ${IDF_LWIP_COMPILE_DEFINITIONS}
         )
 
-        message(STATUS "Adding byteorder functions")
-                
-        # need to read the supplied SDK CONFIG file and replace the appropriate options
-        set(ARCH_CC_FILE "${esp32_idf_SOURCE_DIR}/components/lwip/port/esp32/include/arch/cc.h")       
-        file(READ
-            ${ARCH_CC_FILE}
-            ARCH_CC_CONTENTS)
-
-        string(REPLACE
-            "#endif // BYTE_ORDER"
-            "#endif // __BYTE_ORDER\n\n#define LWIP_DONT_PROVIDE_BYTEORDER_FUNCTIONS\n#define htons(x) __builtin_bswap16(x)\n#define ntohs(x) __builtin_bswap16(x)\n#define htonl(x) __builtin_bswap32(x)\n#define ntohl(x) __builtin_bswap32(x)\n"
-            ARCH_CC_FINAL_CONTENTS
-            "${ARCH_CC_CONTENTS}")
-
-        file(WRITE 
-            ${ARCH_CC_FILE} 
-            "${ARCH_CC_FINAL_CONTENTS}")
-
     endif()
 
     # need to add include path to find our ffconfig.h and target_platform.h

CMake/binutils.common.cmake

@@ -550,7 +550,7 @@ macro(nf_setup_target_build_common)
     # set compile options
     nf_set_compile_options(TARGET ${NANOCLR_PROJECT_NAME}.elf EXTRA_COMPILE_OPTIONS ${NFSTBC_CLR_EXTRA_COMPILE_OPTIONS})
 
-    if(USE_SECURITY_MBEDTLS_OPTION)
+    if(USE_SECURITY_MBEDTLS_OPTION AND NOT RTOS_ESP32_CHECK)
 
         # mbedTLS requires setting a compiler definition in order to pass a config file
         target_compile_definitions(mbedcrypto PUBLIC "-DMBEDTLS_CONFIG_FILE=\"${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/mbedTLS/nf_mbedtls_config.h\"")