add Hybrid Compile

.github/workflows/examples.yml

@@ -20,6 +20,7 @@ jobs:
           - "examples/arduino-wifiscan"
           - "examples/arduino-zigbee-light"
           - "examples/arduino-zigbee-switch"
+          - "examples/tasmota"
           - "examples/espidf-arduino-h2zero-BLE_scan"
           - "examples/espidf-arduino-matter-light"
           - "examples/espidf-arduino-blink"
@@ -49,15 +50,15 @@ jobs:
           python -m pip install --upgrade pip
           pip install wheel
           pip install -U https://github.com/platformio/platformio/archive/develop.zip
-          pio pkg install --global --platform symlink://.
+          pio pkg install --global --platform file://.
         if: "matrix.os == 'windows-2022'"
         env:
           PLATFORMIO_CORE_DIR: C:\plat
           PLATFORMIO_PACKAGES_DIR: C:\plat\pack
           PLATFORMIO_PLATFORMS_DIR: C:\plat\plat
       - name: Build Windows examples
         run: pio run -d ${{ matrix.example }}
-        if: "matrix.os == 'windows-2022'"
+        if: "matrix.example != 'examples/tasmota' && matrix.os == 'windows-2022'"
         env:
           PLATFORMIO_CORE_DIR: C:\plat
           PLATFORMIO_PACKAGES_DIR: C:\plat\pack
@@ -67,8 +68,13 @@ jobs:
           python -m pip install --upgrade pip
           pip install wheel
           pip install -U https://github.com/platformio/platformio/archive/develop.zip
-          pio pkg install --global --platform symlink://.
+          pio pkg install --global --platform file://.
         if: "matrix.os != 'windows-2022'"
+      - name: git clone Tasmota and add to examples
+        run: |
+          git clone -b development --depth 1 https://github.com/arendst/Tasmota.git examples/tasmota
+          cp examples/tasmota_platformio_override.ini examples/tasmota/platformio_override.ini
+        if: "matrix.example == 'examples/tasmota' && matrix.os != 'windows-2022'"
       - name: Build Ubuntu Mac examples
         run: pio run -d ${{ matrix.example }}
         if: "matrix.os != 'windows-2022'"

README.md

@@ -50,7 +50,7 @@ based on devel branch Arduino 3.1.0 and IDF 5.3.1+
 
 ```ini
 [env:development]
-platform = https://github.com/pioarduino/platform-espressif32.git#Arduino/IDF53
+platform = https://github.com/pioarduino/platform-espressif32.git#Arduino/IDF53_work_hybrid
 board = ...
 ...
 ```

boards/esp32-solo1.json

@@ -1,11 +1,8 @@
 {
   "build": {
-    "arduino":{
-      "ldscript": "esp32_out.ld"
-    },
     "core": "esp32",
     "extra_flags": "-DARDUINO_ESP32_DEV -DARDUINO_USB_CDC_ON_BOOT=0 -DCORE32SOLO1",
-    "f_cpu": "80000000L",
+    "f_cpu": "160000000L",
     "f_flash": "40000000L",
     "flash_mode": "dio",
     "mcu": "esp32",
@@ -32,6 +29,11 @@
     "require_upload_port": true,
     "speed": 460800
   },
+  "espidf": {
+    "custom_sdkconfig": [
+      "CONFIG_FREERTOS_UNICORE=y" 
+    ]
+  },
   "url": "https://en.wikipedia.org/wiki/ESP32",
   "vendor": "Espressif"
 }

boards/esp32s3_120_16_8-qio_opi.json

@@ -0,0 +1,50 @@
+{
+  "build": {
+    "arduino": {
+      "memory_type": "qio_opi",
+      "partitions": "default_16MB.csv"
+    },
+    "core": "esp32",
+    "f_cpu": "240000000L",
+    "f_flash": "80000000L",
+    "f_boot": "120000000L",
+    "boot": "qio",
+    "flash_mode": "qio",
+    "extra_flags": [
+      "-DBOARD_HAS_PSRAM",
+      "-DARDUINO_USB_MODE=1",
+      "-DARDUINO_USB_CDC_ON_BOOT=1",
+      "-DARDUINO_RUNNING_CORE=1",
+      "-DARDUINO_EVENT_RUNNING_CORE=1"
+    ],
+    "hwids": [
+      [
+        "0x303A",
+        "0x1001"
+      ]
+    ],
+    "mcu": "esp32s3",
+    "variant": "esp32s3"
+  },
+  "connectivity": [
+    "bluetooth",
+    "wifi"
+  ],
+  "debug": {
+    "openocd_target": "esp32s3.cfg"
+  },
+  "frameworks": [
+    "arduino",
+    "espidf"
+  ],
+  "name": "ESP32-S3 16MB QIO, 8MB OPI PSRAM",
+  "upload": {
+    "flash_size": "16MB",
+    "maximum_ram_size": 327680,
+    "maximum_size": 16777216,
+    "require_upload_port": true,
+    "speed": 460800
+  },
+  "url": "https://www.espressif.com/sites/default/files/documentation/esp32-s3-wroom-1_wroom-1u_datasheet_en.pdf",
+  "vendor": "espressif"
+}

builder/build_lib/CMakeLists.txt

@@ -0,0 +1 @@
+idf_component_register(SRCS "sketch.cpp" "arduino-lib-builder-gcc.c" "arduino-lib-builder-cpp.cpp" "arduino-lib-builder-as.S" INCLUDE_DIRS ".")

builder/build_lib/idf_component.yml

@@ -0,0 +1,12 @@
+dependencies:
+  # Required IDF version
+  idf: ">=5.1"
+  espressif/cmake_utilities:
+    version: "0.*"
+  espressif/fb_gfx:
+    version: "master"
+    path: components/fb_gfx
+    git: https://github.com/espressif/esp32-arduino-lib-builder.git
+    require: public
+    rules:
+      - if: "target in [esp32, esp32s2, esp32s3, esp32p4]"

builder/build_lib/sketch.cpp

@@ -0,0 +1,10 @@
+#include "Arduino.h"
+
+void setup() {
+  Serial.begin(115200);
+}
+
+void loop() {
+  Serial.println("Hello World!");
+  delay(1000);
+}

builder/frameworks/arduino.py

@@ -22,23 +22,179 @@
 http://arduino.cc/en/Reference/HomePage
 """
 
+import subprocess
+import json
+import semantic_version
 import os
+import shutil
 from os.path import join
 
-from SCons.Script import DefaultEnvironment, SConscript
+from SCons.Script import COMMAND_LINE_TARGETS, DefaultEnvironment, SConscript
+from platformio.package.version import pepver_to_semver
+from platformio.project.config import ProjectConfig
+from platformio.package.manager.tool import ToolPackageManager
 
 env = DefaultEnvironment()
+pm = ToolPackageManager()
+platform = env.PioPlatform()
+config = env.GetProjectConfig()
 board = env.BoardConfig()
-build_core = board.get("build.core", "").lower()
+mcu = board.get("build.mcu", "esp32")
+board_sdkconfig = board.get("espidf.custom_sdkconfig", "")
+entry_custom_sdkconfig = "\n"
+flag_custom_sdkconfig = False
+
+if config.has_option("env:"+env["PIOENV"], "custom_sdkconfig"):
+    entry_custom_sdkconfig = env.GetProjectOption("custom_sdkconfig")
+    flag_custom_sdkconfig = True
+
+if len(str(board_sdkconfig)) > 2:
+    flag_custom_sdkconfig = True
+
+extra_flags = (''.join([element for element in board.get("build.extra_flags", "")])).replace("-D", " ")
+framework_reinstall = False
+flag_any_custom_sdkconfig = False
 
 SConscript("_embed_files.py", exports="env")
 
-if "espidf" not in env.subst("$PIOFRAMEWORK"):
-    if os.path.exists(join(DefaultEnvironment().PioPlatform().get_package_dir(
+flag_any_custom_sdkconfig = os.path.exists(join(platform.get_package_dir("framework-arduinoespressif32-libs"),"sdkconfig"))
+
+# Esp32-solo1 libs needs adopted settings
+if flag_custom_sdkconfig == True and ("CORE32SOLO1" in extra_flags or "CONFIG_FREERTOS_UNICORE=y" in entry_custom_sdkconfig or "CONFIG_FREERTOS_UNICORE=y" in board_sdkconfig):
+    if len(str(env.GetProjectOption("build_unflags"))) == 2: # No valid env, needs init
+        env['BUILD_UNFLAGS'] = {}
+    build_unflags = " ".join(env['BUILD_UNFLAGS'])
+    build_unflags = build_unflags + " -mdisable-hardware-atomics -ustart_app_other_cores"
+    new_build_unflags = build_unflags.split()
+    env.Replace(
+      BUILD_UNFLAGS=new_build_unflags
+    )
+
+def install_python_deps():
+    def _get_installed_pip_packages():
+        result = {}
+        packages = {}
+        pip_output = subprocess.check_output(
+            [
+                env.subst("$PYTHONEXE"),
+                "-m",
+                "pip",
+                "list",
+                "--format=json",
+                "--disable-pip-version-check",
+            ]
+        )
+        try:
+            packages = json.loads(pip_output)
+        except:
+            print("Warning! Couldn't extract the list of installed Python packages.")
+            return {}
+        for p in packages:
+            result[p["name"]] = pepver_to_semver(p["version"])
+
+        return result
+
+    deps = {
+        "wheel": ">=0.35.1",
+        "PyYAML": ">=6.0.2",
+        "intelhex": ">=2.3.0"
+    }
+
+    installed_packages = _get_installed_pip_packages()
+    packages_to_install = []
+    for package, spec in deps.items():
+        if package not in installed_packages:
+            packages_to_install.append(package)
+        else:
+            version_spec = semantic_version.Spec(spec)
+            if not version_spec.match(installed_packages[package]):
+                packages_to_install.append(package)
+
+    if packages_to_install:
+        env.Execute(
+            env.VerboseAction(
+                (
+                    '"$PYTHONEXE" -m pip install -U '
+                    + " ".join(
+                        [
+                            '"%s%s"' % (p, deps[p])
+                            for p in packages_to_install
+                        ]
+                    )
+                ),
+                "Installing Arduino Python dependencies",
+            )
+        )
+    return
+
+install_python_deps()
+
+def get_MD5_hash(phrase):
+    import hashlib
+    return hashlib.md5((phrase).encode('utf-8')).hexdigest()[:16]
+
+
+def matching_custom_sdkconfig():
+    # check if current env is matching to existing sdkconfig
+    cust_sdk_is_present = False
+    matching_sdkconfig = False
+    last_sdkconfig_path = join(env.subst("$PROJECT_DIR"),"sdkconfig.defaults")
+    if flag_any_custom_sdkconfig == False:
+        matching_sdkconfig = True
+        return matching_sdkconfig, cust_sdk_is_present
+    if os.path.exists(last_sdkconfig_path) == False:
+        return matching_sdkconfig, cust_sdk_is_present
+    if flag_custom_sdkconfig == False:
+        matching_sdkconfig = False
+        return matching_sdkconfig, cust_sdk_is_present
+    with open(last_sdkconfig_path) as src:
+        line = src.readline()
+        if line.startswith("# TASMOTA__"):
+            cust_sdk_is_present = True;
+            costum_options = entry_custom_sdkconfig
+            if (line.split("__")[1]).strip() == get_MD5_hash((costum_options).strip() + mcu):
+                matching_sdkconfig = True
+
+    return matching_sdkconfig, cust_sdk_is_present
+
+def check_reinstall_frwrk():
+    framework_reinstall = False
+    cust_sdk_is_present = False
+    matching_sdkconfig = False
+    if flag_custom_sdkconfig == True:
+        matching_sdkconfig, cust_sdk_is_present = matching_custom_sdkconfig()
+    if flag_custom_sdkconfig == False and flag_any_custom_sdkconfig == True:
+        # case custom sdkconfig exists and a env without "custom_sdkconfig"
+        framework_reinstall = True
+    if flag_custom_sdkconfig == True  and matching_sdkconfig == False:
+        # check if current custom sdkconfig is different from existing
+        framework_reinstall = True
+    return framework_reinstall
+
+def call_compile_libs():
+    if mcu == "esp32c2":
+        ARDUINO_FRMWRK_C2_LIB_DIR = join(platform.get_package_dir("framework-arduinoespressif32-libs"),mcu)
+        ARDUINO_C2_DIR = join(platform.get_package_dir("framework-arduino-c2-skeleton-lib"),mcu)
+        shutil.copytree(ARDUINO_C2_DIR, ARDUINO_FRMWRK_C2_LIB_DIR, dirs_exist_ok=True)
+    print("*** Compile Arduino IDF libs for %s ***" % env["PIOENV"])
+    SConscript("espidf.py")
+
+if check_reinstall_frwrk() == True:
+    print("*** Reinstall Arduino framework libs ***")
+    shutil.rmtree(platform.get_package_dir("framework-arduinoespressif32-libs"))
+    ARDUINO_FRMWRK_LIB_URL = str(platform.get_package_spec("framework-arduinoespressif32-libs")).split("uri=",1)[1][:-1]
+    pm.install(ARDUINO_FRMWRK_LIB_URL)
+    if flag_custom_sdkconfig == True:
+        call_compile_libs()
+        flag_custom_sdkconfig = False
+
+if flag_custom_sdkconfig == True and flag_any_custom_sdkconfig == False:
+    call_compile_libs()
+
+if "arduino" in env.subst("$PIOFRAMEWORK") and "espidf" not in env.subst("$PIOFRAMEWORK") and env.subst("$ARDUINO_LIB_COMPILE_FLAG") in ("Inactive", "True"):
+    if os.path.exists(join(platform.get_package_dir(
             "framework-arduinoespressif32"), "tools", "platformio-build.py")):
         PIO_BUILD = "platformio-build.py"
     else:
         PIO_BUILD = "pioarduino-build.py"
-    SConscript(
-        join(DefaultEnvironment().PioPlatform().get_package_dir(
-            "framework-arduinoespressif32"), "tools", PIO_BUILD))
+    SConscript(join(platform.get_package_dir("framework-arduinoespressif32"), "tools", PIO_BUILD))