Merge branch 'master' into master

Author: PaulaScharf

.github/codeql/codeql-config.yml

@@ -20,7 +20,9 @@ query-filters:
       problem.severity:
         - recommendation
   - exclude:
-      id: tob/cpp/use-of-legacy-algorithm
+      id: tob/cpp/use-of-legacy-algorithm # We use legacy algorithms in many places for integrity checks
+  - exclude:
+      id: cpp/dead-code-goto # Too many false positives in no-build mode
 
 paths-ignore:
   - tests/**

.github/scripts/process_sarif.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+
+# This script is used to process the SARIF file generated by CodeQL and
+# to rename files back to .ino and adjust line numbers to match the original .ino files.
+
+import json
+import sys
+import os
+
+def process_artifact_location(artifact_location, renamed_files):
+    """
+    Process a single artifact location to rename .cpp files back to .ino
+    """
+    if 'uri' in artifact_location:
+        uri = artifact_location['uri']
+        if uri in renamed_files:
+            print(f"Renaming file: {uri} -> {renamed_files[uri]}")
+            artifact_location['uri'] = renamed_files[uri]
+            return True
+    return False
+
+def process_region(region):
+    """
+    Adjust line numbers in a region by decreasing them by 1
+    """
+    if 'startLine' in region:
+        region['startLine'] = max(1, region['startLine'] - 1)
+    if 'endLine' in region:
+        region['endLine'] = max(1, region['endLine'] - 1)
+
+def process_physical_location(physical_location, renamed_files):
+    """
+    Process a physical location to rename files and adjust line numbers
+    """
+    file_renamed = False
+
+    if 'artifactLocation' in physical_location:
+        if process_artifact_location(physical_location['artifactLocation'], renamed_files):
+            file_renamed = True
+
+    # Adjust line numbers if the file was renamed
+    if file_renamed and 'region' in physical_location:
+        process_region(physical_location['region'])
+
+    return file_renamed
+
+
+def process_sarif_file(sarif_file, renamed_files_file):
+    """
+    Process SARIF file to rename files back to .ino and adjust line numbers
+    """
+    # Read the renamed files mapping
+    with open(renamed_files_file, 'r') as f:
+        renamed_files = json.load(f)
+
+    print(f"Loaded {len(renamed_files)} file mappings:")
+    for cpp_file, ino_file in renamed_files.items():
+        print(f"  {cpp_file} -> {ino_file}")
+
+
+    # Read the SARIF file
+    with open(sarif_file, 'r') as f:
+        sarif_data = json.load(f)
+
+    files_processed = 0
+
+    # Process each run
+    if 'runs' in sarif_data:
+        for run in sarif_data['runs']:
+            # Process results
+            if 'results' in run:
+                for result in run['results']:
+                    # Process all locations in the result
+                    if 'locations' in result:
+                        for location in result['locations']:
+                            if 'physicalLocation' in location:
+                                if process_physical_location(location['physicalLocation'], renamed_files):
+                                    files_processed += 1
+
+                    # Process related locations if they exist
+                    if 'relatedLocations' in result:
+                        for location in result['relatedLocations']:
+                            if 'physicalLocation' in location:
+                                if process_physical_location(location['physicalLocation'], renamed_files):
+                                    files_processed += 1
+
+            # Process artifacts if they exist
+            if 'artifacts' in run:
+                for artifact in run['artifacts']:
+                    if 'location' in artifact and 'uri' in artifact['location']:
+                        uri = artifact['location']['uri']
+                        if uri in renamed_files:
+                            artifact['location']['uri'] = renamed_files[uri]
+                            files_processed += 1
+
+    print(f"Processed {files_processed} file references")
+
+    # Write the processed SARIF file
+    with open(sarif_file, 'w') as f:
+        json.dump(sarif_data, f, indent=2)
+
+def main():
+    if len(sys.argv) != 3:
+        print("Usage: python3 sarif_nobuild.py <sarif_file> <renamed_files_file>")
+        sys.exit(1)
+
+    sarif_file = sys.argv[1]
+    renamed_files_file = sys.argv[2]
+
+    # Check if files exist
+    if not os.path.exists(sarif_file):
+        print(f"SARIF file not found: {sarif_file}")
+        sys.exit(1)
+
+    if not os.path.exists(renamed_files_file):
+        print(f"Renamed files mapping not found: {renamed_files_file}")
+        sys.exit(1)
+
+    try:
+        process_sarif_file(sarif_file, renamed_files_file)
+        print("SARIF file processed successfully")
+    except Exception as e:
+        print(f"Error processing SARIF file: {e}")
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

.github/scripts/set_push_chunks.sh

@@ -78,7 +78,6 @@ chunks+="]"
     echo "build_all=$build_all"
     echo "build_libraries=$BUILD_LIBRARIES"
     echo "build_static_sketches=$BUILD_STATIC_SKETCHES"
-    echo "build_idf=$BUILD_IDF"
     echo "chunk_count=$chunks_count"
     echo "chunks=$chunks"
 } >> "$GITHUB_OUTPUT"

.github/workflows/build_component.yml

@@ -0,0 +1,133 @@
+name: Arduino as ESP-IDF Component
+
+on:
+  workflow_dispatch:
+    inputs:
+      idf_ver:
+        description: "IDF Versions"
+        default: "release-v5.3,release-v5.4,release-v5.5"
+        type: "string"
+        required: true
+      idf_targets:
+        description: "IDF Targets"
+        default: "esp32,esp32s2,esp32s3,esp32c2,esp32c3,esp32c6,esp32h2,esp32p4"
+        type: "string"
+        required: true
+  push:
+    branches:
+      - master
+      - release/*
+  pull_request:
+    paths:
+      - "cores/**"
+      - "libraries/**/*.cpp"
+      - "libraries/**/*.c"
+      - "libraries/**/*.h"
+      - "libraries/**/*.ino"
+      - "libraries/**/ci.json"
+      - "idf_component_examples/**"
+      - "idf_component.yml"
+      - "Kconfig.projbuild"
+      - "CMakeLists.txt"
+      - ".github/workflows/build_component.yml"
+      - ".github/scripts/check-cmakelists.sh"
+      - ".github/scripts/on-push-idf.sh"
+      - ".github/scripts/sketch_utils.sh"
+      - "variants/esp32/**"
+      - "variants/esp32c2/**"
+      - "variants/esp32c3/**"
+      - "variants/esp32c6/**"
+      - "variants/esp32h2/**"
+      - "variants/esp32p4/**"
+      - "variants/esp32s2/**"
+      - "variants/esp32s3/**"
+      - "!*.md"
+      - "!*.txt"
+      - "!*.properties"
+
+concurrency:
+  group: build-component-${{github.event.pull_request.number || github.ref}}
+  cancel-in-progress: true
+
+jobs:
+  cmake-check:
+    name: Check CMakeLists
+    runs-on: ubuntu-latest
+    if: ${{ !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release/')) }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - run: bash ./.github/scripts/check-cmakelists.sh
+
+  set-matrix:
+    name: Set Matrix
+    runs-on: ubuntu-latest
+    if: ${{ !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release/')) }}
+    outputs:
+      idf_ver: ${{ steps.set-matrix.outputs.idf_ver }}
+      idf_target: ${{ steps.set-matrix.outputs.idf_target }}
+    steps:
+      - name: Get IDF Version and Targets
+        id: set-matrix
+        run: |
+          # Default values
+          idf_ver="release-v5.3,release-v5.4,release-v5.5"
+          idf_targets="esp32,esp32s2,esp32s3,esp32c2,esp32c3,esp32c6,esp32h2,esp32p4"
+
+          # Override with inputs if provided
+          if [[ -n "${{ inputs.idf_ver }}" ]]; then
+            idf_ver="${{ inputs.idf_ver }}"
+          fi
+          if [[ -n "${{ inputs.idf_targets }}" ]]; then
+            idf_targets="${{ inputs.idf_targets }}"
+          fi
+
+          # Convert comma-separated strings to JSON arrays using a more robust method
+          idf_ver_json=$(printf '%s\n' "$idf_ver" | tr ',' '\n' | jq -R . | jq -s . | jq -c .)
+          idf_targets_json=$(printf '%s\n' "$idf_targets" | tr ',' '\n' | jq -R . | jq -s . | jq -c .)
+
+          # Debug: Print the JSON for verification
+          echo "Debug - idf_ver_json: $idf_ver_json"
+          echo "Debug - idf_targets_json: $idf_targets_json"
+
+          # Set outputs - ensure no extra whitespace
+          printf "idf_ver=%s\n" "$idf_ver_json" >> $GITHUB_OUTPUT
+          printf "idf_target=%s\n" "$idf_targets_json" >> $GITHUB_OUTPUT
+
+  build-esp-idf-component:
+    name: Build IDF ${{ matrix.idf_ver }} for ${{ matrix.idf_target }}
+    runs-on: ubuntu-latest
+    needs: set-matrix
+    strategy:
+      fail-fast: false
+      matrix:
+        # The version names here correspond to the versions of espressif/idf Docker image.
+        # See https://hub.docker.com/r/espressif/idf/tags and
+        # https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-guides/tools/idf-docker-image.html
+        # for details.
+        idf_ver: ${{ fromJson(needs.set-matrix.outputs.idf_ver) }}
+        idf_target: ${{ fromJson(needs.set-matrix.outputs.idf_target) }}
+    container: espressif/idf:${{ matrix.idf_ver }}
+    steps:
+      - name: Check out arduino-esp32 as a component
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: recursive
+          path: components/arduino-esp32
+
+      - name: Setup jq
+        uses: dcarbone/install-jq-action@e397bd87438d72198f81efd21f876461183d383a # v3.0.1
+
+      - name: Build
+        env:
+          IDF_TARGET: ${{ matrix.idf_target }}
+        shell: bash
+        run: |
+          chmod a+x ./components/arduino-esp32/.github/scripts/*
+          ./components/arduino-esp32/.github/scripts/on-push-idf.sh
+
+      - name: Upload generated sdkconfig files for debugging
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: sdkconfig-${{ matrix.idf_ver }}-${{ matrix.idf_target }}
+          path: ./components/arduino-esp32/idf_component_examples/**/sdkconfig

.github/workflows/codeql.yml

@@ -7,6 +7,10 @@ on:
       - master
   pull_request:
     paths:
+      - "**/*.c"
+      - "**/*.cpp"
+      - "**/*.h"
+      - "**/*.ino"
       - "**/*.py"
       - ".github/workflows/*.yml"
       - ".github/workflows/*.yaml"
@@ -17,19 +21,68 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        language: [python, actions]
+        language: [python, actions, cpp]
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
+      - name: Process .ino files
+        if: matrix.language == 'cpp'
+        run: |
+          # Create a mapping file to track renamed files
+          echo "{}" > renamed_files.json
+
+          # Find all .ino files and process them
+          find . -name "*.ino" -type f | while read -r file; do
+            echo "Processing $file"
+
+            # Get the relative path from repository root
+            rel_path=$(realpath --relative-to=. "$file")
+            cpp_path="${rel_path%.ino}.cpp"
+
+            # Create new .cpp file with Arduino.h include
+            echo "#include <Arduino.h>" > "$cpp_path"
+
+            # Append the original content
+            cat "$file" >> "$cpp_path"
+
+            # Update the mapping file
+            jq --arg ino "$rel_path" --arg cpp "$cpp_path" '. += {($cpp): $ino}' renamed_files.json > temp.json && mv temp.json renamed_files.json
+
+            # Remove the original .ino file
+            rm "$file"
+
+            echo "Converted $file to $cpp_path"
+          done
+
+          echo "Renamed files mapping:"
+          cat renamed_files.json
+
       - name: Initialize CodeQL
         uses: github/codeql-action/init@181d5eefc20863364f96762470ba6f862bdef56b # v3.29.2
         with:
+          build-mode: none
           languages: ${{ matrix.language }}
           config-file: ./.github/codeql/codeql-config.yml
 
       - name: Run CodeQL Analysis
         uses: github/codeql-action/analyze@181d5eefc20863364f96762470ba6f862bdef56b # v3.29.2
         with:
           category: "/language:${{ matrix.language }}"
+          output: sarif-results
+          upload: failure-only
+
+      - name: Process SARIF file
+        if: matrix.language == 'cpp'
+        run: |
+          sarif_file="sarif-results/${{ matrix.language }}.sarif"
+
+          # Run the Python script to process the SARIF file
+          python3 .github/scripts/process_sarif.py "$sarif_file" "renamed_files.json"
+
+      - name: Upload SARIF file
+        uses: github/codeql-action/upload-sarif@181d5eefc20863364f96762470ba6f862bdef56b # v3.29.2
+        with:
+          sarif_file: sarif-results/${{ matrix.language }}.sarif
+          category: "/language:${{ matrix.language }}"