Merge branch 'release/v3.4.0'

Author: valeros

.github/workflows/examples.yml

@@ -7,7 +7,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-16.04, windows-latest, macos-latest]
+        os: [ubuntu-18.04, windows-latest, macos-latest]
         python-version: [3.7]
         example:
           - "examples/arduino-blink"

boards/tinypico.json

@@ -7,7 +7,7 @@
     "extra_flags": "-DARDUINO_TINYPICO -DBOARD_HAS_PSRAM -mfix-esp32-psram-cache-issue",
     "f_cpu": "240000000L",
     "f_flash": "40000000L",
-    "flash_mode": "dio",
+    "flash_mode": "qio",
     "mcu": "esp32",
     "variant": "pico32"
   },

builder/frameworks/_embed_files.py

@@ -125,7 +125,7 @@ def transform_to_asm(target, source, env):
             ),
             suffix=".txt.o",
         ),
-        TxtToAsm=Builder(
+        FileToAsm=Builder(
             action=env.VerboseAction(
                 " ".join(
                     [
@@ -136,7 +136,7 @@ def transform_to_asm(target, source, env):
                         ),
                         "-DDATA_FILE=$SOURCE",
                         "-DSOURCE_FILE=$TARGET",
-                        "-DFILE_TYPE=TEXT",
+                        "-DFILE_TYPE=$FILE_TYPE",
                         "-P",
                         join(
                             env.PioPlatform().get_package_dir("framework-espidf") or "",
@@ -166,7 +166,13 @@ def transform_to_asm(target, source, env):
 
     files = extract_files(flags, files_type)
     if "espidf" in env.subst("$PIOFRAMEWORK"):
-        env.Requires(join("$BUILD_DIR", "${PROGNAME}.elf"), env.TxtToAsm(files))
+        env.Requires(
+            join("$BUILD_DIR", "${PROGNAME}.elf"),
+            env.FileToAsm(
+                files,
+                FILE_TYPE="TEXT" if files_type == "embed_txtfiles" else "BINARY",
+            ),
+        )
     else:
         embed_files(files, files_type)
         remove_config_define(flags, files_type)

builder/frameworks/espidf.py

@@ -53,12 +53,16 @@
 TOOLCHAIN_DIR = platform.get_package_dir(
     "toolchain-%s"
     % (
-        "riscv-esp"
+        "riscv32-esp"
         if mcu == "esp32c3"
-        else ("xtensa32s2" if mcu == "esp32s2" else "xtensa32")
+        else ("xtensa-esp32s2" if mcu == "esp32s2" else "xtensa-esp32")
     )
 )
 
+# Legacy toolchains for mixed IDF/Arduino projects
+if "arduino" in env.subst("$PIOFRAMEWORK"):
+    TOOLCHAIN_DIR = platform.get_package_dir("toolchain-xtensa32")
+
 assert os.path.isdir(FRAMEWORK_DIR)
 assert os.path.isdir(TOOLCHAIN_DIR)
 
@@ -997,7 +1001,8 @@ def _get_installed_pip_packages():
         return result
 
     deps = {
-        "cryptography": ">=2.1.4",
+        # https://github.com/platformio/platform-espressif32/issues/635
+        "cryptography": ">=2.1.4,<35.0.0",
         "future": ">=0.15.2",
         "pyparsing": ">=2.0.3,<2.4.0",
         "kconfiglib": "==13.7.1",
@@ -1348,7 +1353,7 @@ def _skip_prj_source_files(node):
 
 ulp_dir = os.path.join(PROJECT_DIR, "ulp")
 if os.path.isdir(ulp_dir) and os.listdir(ulp_dir) and mcu != "esp32c3":
-    env.SConscript("ulp.py", exports="env project_config idf_variant")
+    env.SConscript("ulp.py", exports="env sdk_config project_config idf_variant")
 
 #
 # Process OTA partition and image

builder/frameworks/ulp.py

@@ -14,12 +14,13 @@
 
 import os
 
-from SCons.Script import Import
-
+from platformio import fs
 from platformio.util import get_systype
 from platformio.proc import where_is_program
 
-Import("env project_config idf_variant")
+from SCons.Script import Import
+
+Import("env sdk_config project_config idf_variant")
 
 ulp_env = env.Clone()
 platform = ulp_env.PioPlatform()
@@ -36,7 +37,8 @@ def prepare_ulp_env_vars(env):
     additional_packages = [
         os.path.join(
             platform.get_package_dir(
-                "toolchain-xtensa%s" % ("32s2" if idf_variant == "esp32s2" else "32")
+                "toolchain-xtensa-esp%s"
+                % ("32s2" if idf_variant == "esp32s2" else "32")
             ),
             "bin",
         ),
@@ -58,8 +60,9 @@ def prepare_ulp_env_vars(env):
 
 def collect_ulp_sources():
     return [
-        os.path.join(ulp_env.subst("$PROJECT_DIR"), "ulp", f)
+        fs.to_unix_path(os.path.join(ulp_env.subst("$PROJECT_DIR"), "ulp", f))
         for f in os.listdir(os.path.join(ulp_env.subst("$PROJECT_DIR"), "ulp"))
+        if f.endswith((".c", ".S", ".s"))
     ]
 
 
@@ -77,6 +80,8 @@ def get_component_includes(target_config):
 
 
 def generate_ulp_config(target_config):
+    riscv_ulp_enabled = sdk_config.get("ESP32S2_ULP_COPROC_RISCV", False)
+
     ulp_sources = collect_ulp_sources()
     cmd = (
         os.path.join(platform.get_package_dir("tool-cmake"), "bin", "cmake"),
@@ -87,15 +92,17 @@ def generate_ulp_config(target_config):
             "components",
             "ulp",
             "cmake",
-            "toolchain-%s-ulp.cmake" % idf_variant,
+            "toolchain-%s-ulp%s.cmake"
+            % (idf_variant, "-riscv" if riscv_ulp_enabled else ""),
         ),
         '-DULP_S_SOURCES="%s"' % ";".join(ulp_sources),
         "-DULP_APP_NAME=ulp_main",
         "-DCOMPONENT_DIR=" + os.path.join(ulp_env.subst("$PROJECT_DIR"), "ulp"),
         '-DCOMPONENT_INCLUDES="%s"' % ";".join(get_component_includes(target_config)),
-        "-DIDF_PATH=" + FRAMEWORK_DIR,
-        "-DSDKCONFIG=" + os.path.join(BUILD_DIR, "config", "sdkconfig.h"),
+        "-DIDF_PATH=" + fs.to_unix_path(FRAMEWORK_DIR),
+        "-DSDKCONFIG_HEADER=" + os.path.join(BUILD_DIR, "config", "sdkconfig.h"),
         "-DPYTHON=" + env.subst("$PYTHONEXE"),
+        "-DULP_COCPU_IS_RISCV=%s" % ("ON" if riscv_ulp_enabled else "OFF"),
         "-GNinja",
         "-B",
         ULP_BUILD_DIR,

builder/main.py

@@ -31,6 +31,15 @@ def _get_board_f_flash(env):
     return str(int(int(frequency) / 1000000)) + "m"
 
 
+def _get_board_flash_mode(env):
+    mode = env.subst("$BOARD_FLASH_MODE")
+    if mode == "qio":
+        return "dio"
+    elif mode == "qout":
+        return "dout"
+    return mode
+
+
 def _parse_size(value):
     if isinstance(value, int):
         return value
@@ -71,8 +80,13 @@ def _parse_partitions(env):
                 "flags": tokens[5] if len(tokens) > 5 else None
             }
             result.append(partition)
-            next_offset = (_parse_size(partition['offset']) +
-                           _parse_size(partition['size']))
+            next_offset = _parse_size(partition["offset"]) + _parse_size(
+                partition["size"]
+            )
+
+            bound = 0x10000 if partition["type"] in ("0", "app") else 4
+            next_offset = (next_offset + bound - 1) & ~(bound - 1)
+
     return result
 
 
@@ -130,6 +144,7 @@ def __fetch_spiffs_size(target, source, env):
 
 env.Replace(
     __get_board_f_flash=_get_board_f_flash,
+    __get_board_flash_mode=_get_board_flash_mode,
 
     AR="%s-elf-ar" % toolchain_arch,
     AS="%s-elf-as" % toolchain_arch,
@@ -302,7 +317,7 @@ def __fetch_spiffs_size(target, source, env):
             "--before", "default_reset",
             "--after", "hard_reset",
             "write_flash", "-z",
-            "--flash_mode", "$BOARD_FLASH_MODE",
+            "--flash_mode", "${__get_board_flash_mode(__env__)}",
             "--flash_freq", "${__get_board_f_flash(__env__)}",
             "--flash_size", "detect"
         ],

examples/espidf-aws-iot/components/esp-aws-iot/aws-iot-device-sdk-embedded-C/PortingGuide.md

@@ -76,8 +76,8 @@ For additional details about API parameters refer to the [API documentation](htt
 Define the `TLSDataParams` Struct as in `network_platform.h`
 This is used for data specific to the TLS library being used.
 
-`IoT_Error_t iot_tls_init(Network *pNetwork, char *pRootCALocation, const char *pDeviceCertLocation,
-  						 const char *pDevicePrivateKeyLocation, const char *pDestinationURL,
+`IoT_Error_t iot_tls_init(Network *pNetwork, char *pRootCALocation, char *pDeviceCertLocation,
+  						 char *pDevicePrivateKeyLocation, char *pDestinationURL,
   						 uint16_t DestinationPort, uint32_t timeout_ms, bool ServerVerificationFlag);`
 Initialize the network client / structure.
 

examples/espidf-aws-iot/components/esp-aws-iot/aws-iot-device-sdk-embedded-C/README.md

@@ -1,27 +1,11 @@
-**We have released version 4.0.0 beta 1 of this SDK on the [v4_beta](https://github.com/aws/aws-iot-device-sdk-embedded-C/tree/v4_beta) branch and encourage everyone to give it a try.**
+# AWS IoT Device SDK for Embedded C
 
-Version 4 is a new design, and therefore **NOT** backwards compatible with version 3.0.1. We will continue to fix bugs in v3.0.1 even after v4.0.0 is released, but we may not add new features to v3.0.1.
+This tag [3.1.5](https://github.com/aws/aws-iot-device-sdk-embedded-C/tree/v3.1.5) contains the v3 version of AWS IoT Device SDK for Embedded C. No new features will be added to this tag; instead, only bug fixes will be made and minimally tested.
 
-Please be aware that v4 beta may have bugs and performance issues. Additionally, there are currently missing features compared to v3.0.1. See the [README](https://github.com/aws/aws-iot-device-sdk-embedded-C/blob/v4_beta/README.md/) on the v4_beta branch for more information.
-
-## Branches
-
-### Master branch
-The [master](https://github.com/aws/aws-iot-device-sdk-embedded-C/tree/master) branch will now contain bug fixes/features that have been minimally tested to ensure nothing major is broken. The current version on the master branch is v3.0.1. Eventually, we will move v4.0.0 to the master branch and move v3.0.1 to a legacy branch.
-
-### Release branch
-The [release](https://github.com/aws/aws-iot-device-sdk-embedded-C/tree/release) branch will contain new releases for the SDK that have been tested thoroughly on all supported platforms. Please ensure that you are tracking the release branch for all production work. The current version on the release branch is v3.0.1. Eventually, we will move v4.0.0 to the release branch and move v3.0.1 to a legacy branch.
-
-### v4_beta branch
-The [v4_beta](https://github.com/aws/aws-iot-device-sdk-embedded-C/tree/v4_beta) branch will contain new features and a new design that inherits from both the AWS IoT Device SDK Embedded C and the libraries provided with Amazon FreeRTOS. This is version 4.0.0 of the SDK. Please be aware that v4 beta may have bugs and performance issues. Eventually, we will move v4.0.0 to the master/release branches and delete v4 beta branch.
-
-### Development branch
-
-The [development](https://github.com/aws/aws-iot-device-sdk-embedded-C/tree/development) currently hosts development of the next iteration of the AWS IoT Embedded C SDK version 4. It is currently a work in progress and should not be used to create any products.  We will update this README when that status changes.
 
 ## Overview
 
-The AWS IoT device SDK for embedded C is a collection of C source files which can be used in embedded applications to securely connect to the [AWS IoT platform](http://docs.aws.amazon.com/iot/latest/developerguide/what-is-aws-iot.html). It includes transport clients **MQTT**, **TLS** implementations and examples for their use. It also supports AWS IoT specific features such as **Thing Shadow**. It is distributed in source form and intended to be built into customer firmware along with application code, other libraries and RTOS. For additional information about porting the Device SDK for embedded C onto additional platforms please refer to the [PortingGuide](https://github.com/aws/aws-iot-device-sdk-embedded-c/blob/master/PortingGuide.md/).
+The AWS IoT device SDK for embedded C is a collection of C source files which can be used in embedded applications to securely connect to the [AWS IoT platform](http://docs.aws.amazon.com/iot/latest/developerguide/what-is-aws-iot.html). It includes transport clients **MQTT**, **TLS** implementations and examples for their use. It also supports AWS IoT specific features such as **Thing Shadow**. It is distributed in source form and intended to be built into customer firmware along with application code, other libraries and RTOS. For additional information about porting the Device SDK for embedded C onto additional platforms please refer to the [PortingGuide](./PortingGuide.md).
 
 ## Features
 The Device SDK simplifies access to the Pub/Sub functionality of the AWS IoT broker via MQTT and provide APIs to interact with Thing Shadows. The SDK has been tested to work with the AWS IoT platform to ensure best interoperability of a device with the AWS IoT platform.

examples/espidf-aws-iot/components/esp-aws-iot/aws-iot-device-sdk-embedded-C/doc/architecture.png

@@ -0,0 +1,32 @@
+/**
+@page architecture SDK Architecture and Design
+@brief This page documents the architecture and design goals of this SDK.
+
+The Embedded C SDK has a layered architecture. It is currently based on the MQTT protocol, but may support protocols like HTTP in the future.
+
+This SDK is primarily targeted for resource-constrained devices.
+
+@section architecture_considerations Considerations during design
+- Low memory footprint (around 6k of code size without TLS library for the MQTT layer)
+- Flexibility in picking and choosing functionality
+- Static memory only (no malloc)
+- Configurable resource usage (JSON tokens, MQTT subscription handlers, etc.)
+- Portability across RTOSes
+
+@section architecture_diagram Layered architecture
+@brief There are primarily 3 layers in the SDK as shown in the diagram below.
+
+@image html architecture.png
+
+@subsection architecture_tls TLS layer
+
+This SDK uses TLS sockets to talk to the server. We provide wrappers around mbed TLS (ARM) that allow the libraries to connect with AWS IoT Platform.
+
+@subsection architecture_mqtt AWS IoT MQTT client
+
+The MQTT library included in this SDK is a modification of the [Paho C MQTT client](https://www.eclipse.org/paho/clients/c/). MQTT is a standardized publish subscribe protocol popular on embedded devices. AWS IoT supports the MQTT protocol for communication with devices. This MQTT library can be used standalone or with other AWS IoT services such as Thing Shadows or Jobs.
+
+@subsection architecture_shadow Thing Shadow
+
+This SDK provides a client implementation for [AWS IoT Thing Shadows](https://docs.aws.amazon.com/iot/latest/developerguide/iot-device-shadows.html). A device's Shadow is a JSON document that is used to store and retrieve its current state information in the cloud regardless of whether it is connected to the Internet.
+*/