Update workflows so they use the Dockerfile too

Author: PaulZC

.github/workflows/compile-rtk-firmware.yml

@@ -6,8 +6,7 @@ on:
 env:
   FILENAME_PREFIX: RTK_Surveyor_Firmware
   FIRMWARE_VERSION_MAJOR: 4
-  FIRMWARE_VERSION_MINOR: 3
-  POINTPERFECT_TOKEN: ${{ secrets.POINTPERFECT_TOKEN }}
+  FIRMWARE_VERSION_MINOR: 4
   #We limit the ESP32 core to v2.0.2
   CORE_VERSION: 2.0.2
 
@@ -52,114 +51,54 @@ jobs:
               echo "DEBUG_LEVEL=debug" >> "$GITHUB_ENV"
           fi
 
-      - name: Setup Arduino CLI
-        uses: arduino/setup-arduino-cli@v1
-
-      - name: Start config file
-        run: arduino-cli config init --additional-urls "https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_index.json"
-
-      - name: Update index
-        run: arduino-cli core update-index
-
-      - name: Install platform
-        run: arduino-cli core install esp32:esp32@${{ env.CORE_VERSION }}
-
-      - name: Get Known Libraries
-        run: arduino-cli lib install
-          [email protected]
-          [email protected]
-          [email protected]
-          "ESP32-OTA-Pull"@1.0.0
-          [email protected]
-          [email protected]
-          [email protected]
-          "SdFat"@2.1.1
-          "SparkFun LIS2DH12 Arduino Library"@1.0.3
-          "SparkFun MAX1704x Fuel Gauge Arduino Library"@1.0.4
-          "SparkFun u-blox GNSS v3"@3.0.14
-          [email protected]
-          [email protected]
-
-      - name: Enable external libs
-        run: arduino-cli config set library.enable_unsafe_install true
-
-      - name: Get Libraries
-        run: arduino-cli lib install --git-url
-          https://github.com/sparkfun/SparkFun_Qwiic_OLED_Arduino_Library.git
-          https://github.com/me-no-dev/ESPAsyncWebServer.git 
-          https://github.com/me-no-dev/AsyncTCP.git
-          
-      #Incorporate ESP-Now patch into core: https://github.com/espressif/arduino-esp32/pull/7044/files
-      #- name: Patch ESP32 Core
-      #  run: |
-      #    cd Firmware/RTK_Surveyor/Patch/
-      #    cp WiFiGeneric.cpp /home/runner/.arduino15/packages/esp32/hardware/esp32/${{ env.CORE_VERSION }}/libraries/WiFi/src/WiFiGeneric.cpp
-
-      #Patch Server.h to avoid https://github.com/arduino-libraries/Ethernet/issues/88#issuecomment-455498941
-      #Note: this patch can be removed if/when we upgrade to ESP32 core >= v2.0.6 
-      - name: Patch ESP32 Server.h for Ethernet
-        run: |
-          cd Firmware/RTK_Surveyor/Patch/
-          cp Server.h /home/runner/.arduino15/packages/esp32/hardware/esp32/${{ env.CORE_VERSION }}/cores/esp32/Server.h
-
-      - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-
-      # Configure Python - now we have Python installed, we need to provide everything needed by esptool otherwise the compile fails
-      - name: Configure Python
+      - name: Run Dockerfile and copy files
+        # The --quiet suppresses the build output, keeping the secrets safe
         run: |
-          pip3 install pyserial
-
-      - name: Update index_html
-        run: |
-          cd Firmware/Tools
-          python index_html_zipper.py ../RTK_Surveyor/AP-Config/index.html ../RTK_Surveyor/form.h
-
-      - name: Update main_js
-        run: |
-          cd Firmware/Tools
-          python main_js_zipper.py ../RTK_Surveyor/AP-Config/src/main.js ../RTK_Surveyor/form.h
+          cd ./Firmware
+          echo "*** The docker build is quiet to protect the GitHub secrets ***"
+          docker build -t rtk_firmware --no-cache --quiet \
+            --build-arg CORE_VERSION=${{ env.CORE_VERSION }} \
+            --build-arg FIRMWARE_VERSION_MAJOR=${{ env.FIRMWARE_VERSION_MAJOR }} \
+            --build-arg FIRMWARE_VERSION_MINOR=${{ env.FIRMWARE_VERSION_MINOR }} \
+            --build-arg POINTPERFECT_TOKEN=${{ secrets.POINTPERFECT_TOKEN }} \
+            --build-arg ENABLE_DEVELOPER=${{ env.ENABLE_DEVELOPER }} \
+            --build-arg DEBUG_LEVEL=${{ env.DEBUG_LEVEL }} \
+            .
+          docker create --name=rtk_container rtk_firmware:latest
+          mkdir ./build
+          docker cp rtk_container:/RTK_Surveyor.ino.bin ./build
+          docker cp rtk_container:/RTK_Surveyor.ino.elf ./build
+          docker cp rtk_container:/RTK_Surveyor/form.h ./RTK_Surveyor
+          docker container rm rtk_container
 
       - name: Commit and push form.h
-        uses: actions-js/push@master
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          directory: ./Firmware/RTK_Surveyor
-          branch: ${{ env.BRANCH }}
-          message: 'Update form.h via Python'
-
-      - name: Copy custom app3M_fat9M_16MB.csv
-        run:
-          cp Firmware/app3M_fat9M_16MB.csv /home/runner/.arduino15/packages/esp32/hardware/esp32/${{ env.CORE_VERSION }}/tools/partitions/app3M_fat9M_16MB.csv
-
-      - name: Compile Sketch
-        run: arduino-cli compile --fqbn "esp32:esp32:esp32":DebugLevel=${{ env.DEBUG_LEVEL }} ./Firmware/RTK_Surveyor/RTK_Surveyor.ino
-          --build-property build.partitions=app3M_fat9M_16MB
-          --build-property upload.maximum_size=3145728 
-          --build-property "compiler.cpp.extra_flags=\"-DPOINTPERFECT_TOKEN=$POINTPERFECT_TOKEN\" \"-DFIRMWARE_VERSION_MAJOR=$FIRMWARE_VERSION_MAJOR\" \"-DFIRMWARE_VERSION_MINOR=$FIRMWARE_VERSION_MINOR\" \"-DENABLE_DEVELOPER=${{ env.ENABLE_DEVELOPER }}\""
-          --export-binaries
+        run: |
+          git config --local user.email "github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git add ./Firmware/RTK_Surveyor/form.h
+          git commit -m "Update form.h via Python"
+          git push origin ${{ env.BRANCH }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Rename binary
         run: |
-          cd Firmware/RTK_Surveyor/build/esp32.esp32.esp32/
+          cd ./Firmware/build/
           mv RTK_Surveyor.ino.bin ${{ env.FILENAME_PREFIX }}${{ env.FILE_ENDING_UNDERSCORE }}.bin
 
       - name: Upload binary to action
         uses: actions/upload-artifact@v4
         with: 
           name: ${{ env.FILENAME_PREFIX }}${{ env.FILE_ENDING_UNDERSCORE }}
-          path: ./Firmware/RTK_Surveyor/build/esp32.esp32.esp32/${{ env.FILENAME_PREFIX }}${{ env.FILE_ENDING_UNDERSCORE }}.bin
-
+          path: ./Firmware/build/${{ env.FILENAME_PREFIX }}${{ env.FILE_ENDING_UNDERSCORE }}.bin
 
       - name: Push binary to Binaries Repo
         # uses: dmnemec/copy_file_to_another_repo_action #Workaround for Issue: https://github.com/orgs/community/discussions/55820#discussioncomment-5946136
         uses: Jason2866/copy_file_to_another_repo_action@http408_fix
         env:
           API_TOKEN_GITHUB: ${{ secrets.API_GITHUB_RTK_FILE_TOKEN }}
         with:
-          source_file: ./Firmware/RTK_Surveyor/build/esp32.esp32.esp32/${{ env.FILENAME_PREFIX }}${{ env.FILE_ENDING_UNDERSCORE }}.bin
+          source_file: ./Firmware/build/${{ env.FILENAME_PREFIX }}${{ env.FILE_ENDING_UNDERSCORE }}.bin
           destination_repo: 'sparkfun/SparkFun_RTK_Firmware_Binaries'
           destination_folder: ''
           user_email: '[email protected]'

.github/workflows/non-release-build.yml

@@ -1,4 +1,4 @@
-name: RTK Firmware Non-Release Build
+name: Non-Release Build
 on: 
   workflow_dispatch:
     branches: 
@@ -7,7 +7,6 @@ env:
   FILENAME_PREFIX: RTK_Surveyor_Firmware
   FIRMWARE_VERSION_MAJOR: 99
   FIRMWARE_VERSION_MINOR: 99
-  POINTPERFECT_TOKEN: ${{ secrets.POINTPERFECT_TOKEN }}
   #We limit the ESP32 core to v2.0.2
   CORE_VERSION: 2.0.2
 
@@ -43,105 +42,43 @@ jobs:
             echo "ENABLE_DEVELOPER=true" >> "$GITHUB_ENV"
             echo "DEBUG_LEVEL=debug" >> "$GITHUB_ENV"
 
-      - name: Setup Arduino CLI
-        uses: arduino/setup-arduino-cli@v1
-
-      - name: Start config file
-        run: arduino-cli config init --additional-urls "https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_index.json"
-
-      - name: Update index
-        run: arduino-cli core update-index
-
-      - name: Update library index
-        run: arduino-cli lib update-index
-
-      - name: Install platform
-        run: arduino-cli core install esp32:esp32@${{ env.CORE_VERSION }}
-
-      - name: Get Known Libraries
-        run: arduino-cli lib install
-          [email protected]
-          [email protected]
-          [email protected]
-          "ESP32-OTA-Pull"@1.0.0
-          [email protected]
-          [email protected]
-          [email protected]
-          "SdFat"@2.1.1
-          "SparkFun LIS2DH12 Arduino Library"@1.0.3
-          "SparkFun MAX1704x Fuel Gauge Arduino Library"@1.0.4
-          "SparkFun u-blox GNSS v3"@3.0.14
-          [email protected]
-          [email protected]
-
-      - name: Enable external libs
-        run: arduino-cli config set library.enable_unsafe_install true
-
-      - name: Get Libraries
-        run: arduino-cli lib install --git-url
-          https://github.com/sparkfun/SparkFun_Qwiic_OLED_Arduino_Library.git
-          https://github.com/me-no-dev/ESPAsyncWebServer.git 
-          https://github.com/me-no-dev/AsyncTCP.git
-          
-      #Incorporate ESP-Now patch into core: https://github.com/espressif/arduino-esp32/pull/7044/files
-      #- name: Patch ESP32 Core
-      #  run: |
-      #    cd Firmware/RTK_Surveyor/Patch/
-      #    cp WiFiGeneric.cpp /home/runner/.arduino15/packages/esp32/hardware/esp32/${{ env.CORE_VERSION }}/libraries/WiFi/src/WiFiGeneric.cpp
-
-      #Patch Server.h to avoid https://github.com/arduino-libraries/Ethernet/issues/88#issuecomment-455498941
-      #Note: this patch can be removed if/when we upgrade to ESP32 core >= v2.0.6 
-      - name: Patch ESP32 Server.h for Ethernet
-        run: |
-          cd Firmware/RTK_Surveyor/Patch/
-          cp Server.h /home/runner/.arduino15/packages/esp32/hardware/esp32/${{ env.CORE_VERSION }}/cores/esp32/Server.h
-
-      - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-
-      # Configure Python - now we have Python installed, we need to provide everything needed by esptool otherwise the compile fails
-      - name: Configure Python
+      - name: Run Dockerfile and copy files
+        # The --quiet suppresses the build output, keeping the secrets safe
         run: |
-          pip3 install pyserial
-
-      - name: Update index_html
-        run: |
-          cd Firmware/Tools
-          python index_html_zipper.py ../RTK_Surveyor/AP-Config/index.html ../RTK_Surveyor/form.h
-
-      - name: Update main_js
-        run: |
-          cd Firmware/Tools
-          python main_js_zipper.py ../RTK_Surveyor/AP-Config/src/main.js ../RTK_Surveyor/form.h
+          cd ./Firmware
+          echo "*** The docker build is quiet to protect the GitHub secrets ***"
+          docker build -t rtk_firmware --no-cache --quiet \
+            --build-arg CORE_VERSION=${{ env.CORE_VERSION }} \
+            --build-arg FIRMWARE_VERSION_MAJOR=${{ env.FIRMWARE_VERSION_MAJOR }} \
+            --build-arg FIRMWARE_VERSION_MINOR=${{ env.FIRMWARE_VERSION_MINOR }} \
+            --build-arg POINTPERFECT_TOKEN=${{ secrets.POINTPERFECT_TOKEN }} \
+            --build-arg ENABLE_DEVELOPER=${{ env.ENABLE_DEVELOPER }} \
+            --build-arg DEBUG_LEVEL=${{ env.DEBUG_LEVEL }} \
+            .
+          docker create --name=rtk_container rtk_firmware:latest
+          mkdir ./build
+          docker cp rtk_container:/RTK_Surveyor.ino.bin ./build
+          docker cp rtk_container:/RTK_Surveyor.ino.elf ./build
+          docker cp rtk_container:/RTK_Surveyor/form.h ./RTK_Surveyor
+          docker container rm rtk_container
 
       - name: Commit and push form.h
-        uses: actions-js/push@master
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          directory: ./Firmware/RTK_Surveyor
-          branch: ${{ env.BRANCH }}
-          message: 'Update form.h via Python'
-
-      - name: Copy custom app3M_fat9M_16MB.csv
-        run:
-          cp Firmware/app3M_fat9M_16MB.csv /home/runner/.arduino15/packages/esp32/hardware/esp32/${{ env.CORE_VERSION }}/tools/partitions/app3M_fat9M_16MB.csv
-
-      - name: Compile Sketch
-        run: arduino-cli compile --fqbn "esp32:esp32:esp32":DebugLevel=${{ env.DEBUG_LEVEL }} ./Firmware/RTK_Surveyor/RTK_Surveyor.ino
-          --build-property build.partitions=app3M_fat9M_16MB
-          --build-property upload.maximum_size=3145728 
-          --build-property "compiler.cpp.extra_flags=\"-DPOINTPERFECT_TOKEN=$POINTPERFECT_TOKEN\" \"-DFIRMWARE_VERSION_MAJOR=$FIRMWARE_VERSION_MAJOR\" \"-DFIRMWARE_VERSION_MINOR=$FIRMWARE_VERSION_MINOR\" \"-DENABLE_DEVELOPER=${{ env.ENABLE_DEVELOPER }}\""
-          --export-binaries
+        run: |
+          git config --local user.email "github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git add ./Firmware/RTK_Surveyor/form.h
+          git commit -m "Update form.h via Python"
+          git push origin ${{ env.BRANCH }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Create artifact name
         run: |
           echo "ARTIFACT=${{ env.FILENAME_PREFIX }}${{ env.FILE_ENDING_UNDERSCORE }}" >> $GITHUB_ENV
 
       - name: Create artifact directory
         run: |
-          cd Firmware/RTK_Surveyor/build/esp32.esp32.esp32/
+          cd ./Firmware/build/
           mkdir ${{ env.ARTIFACT }}
           mv RTK_Surveyor.ino.bin ${{ env.ARTIFACT }}
           mv RTK_Surveyor.ino.elf ${{ env.ARTIFACT }}
@@ -150,5 +87,5 @@ jobs:
         uses: actions/upload-artifact@v4
         with: 
           name: ${{ env.ARTIFACT }}
-          path: Firmware/RTK_Surveyor/build/esp32.esp32.esp32/${{ env.ARTIFACT }}
+          path: ./Firmware/build/${{ env.ARTIFACT }}
           retention-days: 7

Firmware/Dockerfile

@@ -2,20 +2,20 @@ FROM ubuntu:latest AS upstream
 
 ARG DEBIAN_FRONTEND=noninteractive
 
-ENV FILENAME_PREFIX=RTK_Firmware
-ENV FIRMWARE_VERSION_MAJOR=99
-ENV FIRMWARE_VERSION_MINOR=99
-ENV CORE_VERSION=2.0.2
+ARG CORE_VERSION=2.0.2
+
+ARG FIRMWARE_VERSION_MAJOR=99
+ARG FIRMWARE_VERSION_MINOR=99
 
 # ESP32 Core Debug Level
 # We use "none" for releases and "error" for release_candidate
 # You may find "verbose" useful while you are debugging your changes
-ENV DEBUG_LEVEL=error
+ARG DEBUG_LEVEL=error
 
 # Developer Mode
 # You may find "true" useful while you are making changes
 # Set to false for releases
-ENV ENABLE_DEVELOPER=true
+ARG ENABLE_DEVELOPER=true
 
 # If you have your own u-blox PointPerfect token, define it here
 # or pass it in as an arg when building the Dockerfile
@@ -75,45 +75,37 @@ RUN arduino-cli config set library.enable_unsafe_install true
 RUN arduino-cli lib install --git-url https://github.com/me-no-dev/ESPAsyncWebServer.git
 RUN arduino-cli lib install --git-url https://github.com/me-no-dev/AsyncTCP.git
 
-# Copy RTK_Everywhere into /work and build deployment image
+# Copy RTK_Surveyor and build deployment image
 FROM upstream AS deployment
 
-# Create work directory
-WORKDIR /work
+# Add the source files
 ADD . .
 
-# Get current date
-#RUN echo "$(date +'%b_%d_%Y')" > date_scores.txt
-#RUN DATE_SCORES=$(cat date_scores.txt) && echo "Date: ${DATE_SCORES}"
-#RUN echo "$(date +'%b %d %Y')" > date_no_scores.txt
-#RUN DATE_NO_SCORES=$(cat date_no_scores.txt) && echo "Date: ${DATE_NO_SCORES}"
-
 # Patch Server.h to avoid https://github.com/arduino-libraries/Ethernet/issues/88#issuecomment-455498941
-WORKDIR /work/RTK_Surveyor/Patch
-RUN cp Server.h "/root/.arduino15/packages/esp32/hardware/esp32/${CORE_VERSION}/cores/esp32/Server.h"
+RUN cd ./RTK_Surveyor/Patch \
+    && cp Server.h "/root/.arduino15/packages/esp32/hardware/esp32/${CORE_VERSION}/cores/esp32/Server.h"
 
 # Update form.h with index_html and main_js
-WORKDIR /work/Tools
-RUN python index_html_zipper.py ../RTK_Surveyor/AP-Config/index.html ../RTK_Surveyor/form.h
-RUN python main_js_zipper.py ../RTK_Surveyor/AP-Config/src/main.js ../RTK_Surveyor/form.h
+RUN cd ./Tools \
+    && python index_html_zipper.py ../RTK_Surveyor/AP-Config/index.html ../RTK_Surveyor/form.h \
+    && python main_js_zipper.py ../RTK_Surveyor/AP-Config/src/main.js ../RTK_Surveyor/form.h
 
 # Copy custom app3M_fat9M_16MB.csv
-WORKDIR /work
 RUN cp app3M_fat9M_16MB.csv "/root/.arduino15/packages/esp32/hardware/esp32/${CORE_VERSION}/tools/partitions/app3M_fat9M_16MB.csv"
 
 # Compile Sketch
-WORKDIR /work/RTK_Surveyor
-RUN arduino-cli compile --fqbn "esp32:esp32:esp32":DebugLevel=${DEBUG_LEVEL} \
+RUN cd ./RTK_Surveyor \
+    && arduino-cli compile --fqbn "esp32:esp32:esp32":DebugLevel=${DEBUG_LEVEL} \
     ./RTK_Surveyor.ino \
     --build-property build.partitions=app3M_fat9M_16MB \
     --build-property upload.maximum_size=3145728 \
-    --build-property "compiler.cpp.extra_flags=\"-DPOINTPERFECTTOKEN=${POINTPERFECT_TOKEN}\" \
+    --build-property "compiler.cpp.extra_flags=\"-DPOINTPERFECT_TOKEN=${POINTPERFECT_TOKEN}\" \
     \"-DFIRMWARE_VERSION_MAJOR=${FIRMWARE_VERSION_MAJOR}\" \
     \"-DFIRMWARE_VERSION_MINOR=${FIRMWARE_VERSION_MINOR}\" \
     \"-DENABLE_DEVELOPER=${ENABLE_DEVELOPER}\"" \
     --export-binaries
 
 # Copy the compile output. List the files
 FROM deployment AS output
-COPY --from=deployment /work/RTK_Surveyor/build/esp32.esp32.esp32 /
+COPY --from=deployment ./RTK_Surveyor/build/esp32.esp32.esp32 /
 CMD echo $(ls /*.*)