build: use qemu from espressif rather than custom built

Also use bookworm-slim, minimize docker layers

Author: Lawrence Nahum

.gitlab-ci.yml

@@ -7,7 +7,7 @@ stages:
   - test
   - deploy
 
-image: blockstream/verde@sha256:391a4337c174f2eb47dc9f31d7e25d19a5f101bbc07fa8a296866c1ee52b8473
+image: blockstream/verde@sha256:9171360dae48d3fba7c0f36ab5eb45e2529d178a603e89197cb08fb123ca619e
 
 test_bip85_rsa_key_gen:
   tags:

Dockerfile

@@ -1,34 +1,10 @@
-FROM debian:bookworm@sha256:321341744acb788e251ebd374aecc1a42d60ce65da7bd4ee9207ff6be6686a62 AS base
-RUN dpkg --add-architecture i386 && apt-get update -qq && apt-get upgrade --no-install-recommends --no-install-suggests -yqq && apt-get install --no-install-recommends --no-install-suggests -yqq git wget libncurses-dev flex bison gperf libffi-dev libssl-dev dfu-util cmake ninja-build build-essential ca-certificates ccache curl make pkg-config python3 python3-dev python3-pip python3-setuptools python3-serial python3-click python3-cryptography python3-future python3-pyparsing python3-pyelftools python3-pkg-resources python3-wheel python3-venv python3-sphinx unzip bluez-tools bluez libusb-1.0-0 clang lld clang-format libglib2.0-dev libpixman-1-dev libgcrypt20-dev virtualenv libslirp-dev gcc-multilib libc6-dev-i386 libsdl2-dev:i386 libcurl4-openssl-dev:i386 && apt-get -yqq autoremove && apt-get -yqq clean && rm -rf /var/lib/apt/lists/* /var/cache/* /tmp/*
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
-RUN python3 -m pip install --break-system-packages --user pycodestyle
-
-FROM base AS esp-idf
+FROM debian:bookworm-slim@sha256:12c396bd585df7ec21d5679bb6a83d4878bc4415ce926c9e5ea6426d23c60bdc
 
 # These ARGs are easily parseable (eg by HWI)
 ARG ESP_IDF_BRANCH=v5.4
 ARG ESP_IDF_COMMIT=67c1de1eebe095d554d281952fde63c16ee2dca0
-RUN mkdir ~/esp && cd ~/esp && git clone --quiet --depth=1 --branch ${ESP_IDF_BRANCH} --single-branch --recursive --shallow-submodules https://github.com/espressif/esp-idf.git
-RUN cd ~/esp/esp-idf && git checkout ${ESP_IDF_COMMIT} && ./install.sh esp32 esp32s3
-
-FROM base AS esp-qemu
-
-# These ARGs are easily parseable (eg by HWI)
-ARG ESP_QEMU_BRANCH=esp-develop-9.0.0-20240606
-ARG ESP_QEMU_COMMIT=abb5ce24386972e048b401f9eca10e90b8427a20
-RUN git clone --quiet --depth 1 --branch ${ESP_QEMU_BRANCH} --single-branch --shallow-submodules https://github.com/espressif/qemu.git \
-&& cd qemu && git checkout ${ESP_QEMU_COMMIT}
 
-RUN cd qemu && ./configure --target-list=xtensa-softmmu --prefix=/opt \
-    --enable-gcrypt \
-    --disable-sanitizers \
-    --disable-strip --disable-user \
-    --disable-capstone --disable-vnc \
-    --disable-sdl --disable-gtk --enable-slirp \
-    && ninja -C build install && rm -fr /qemu
-
-FROM esp-idf
-COPY --from=esp-qemu /opt /opt
-COPY requirements.txt /
 SHELL ["/bin/bash", "-c"]
-RUN virtualenv -p python3 /venv && source /venv/bin/activate && pip install --require-hashes -r /requirements.txt
+COPY requirements.txt /
+
+RUN dpkg --add-architecture i386 && apt-get update -qq && apt-get upgrade --no-install-recommends --no-install-suggests -yqq && apt-get install --no-install-recommends --no-install-suggests -yqq git wget libncurses-dev flex bison gperf libffi-dev libssl-dev dfu-util cmake ninja-build build-essential ca-certificates ccache curl make pkg-config python3 python3-dev python3-pip python3-setuptools python3-serial python3-click python3-cryptography python3-future python3-pyparsing python3-pyelftools python3-pkg-resources python3-wheel python3-venv python3-sphinx unzip bluez-tools bluez libusb-1.0-0 clang lld clang-format libglib2.0 libpixman-1-0 libsdl2-2.0-0 libgcrypt20-dev virtualenv libslirp0 gcc-multilib libc6-dev-i386 libsdl2-dev:i386 libcurl4-openssl-dev:i386 libmbedtls-dev:i386 && apt-get -yqq autoremove && apt-get -yqq clean && rm -rf /var/lib/apt/lists/* /var/cache/* /tmp/* /usr/share/doc /usr/share/man /usr/share/info /usr/share/locale /usr/share/common-licenses && update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && python3 -m pip install --break-system-packages --user pycodestyle && mkdir ~/esp && cd ~/esp && git clone --quiet --depth=1 --branch ${ESP_IDF_BRANCH} --single-branch --recursive --shallow-submodules https://github.com/espressif/esp-idf.git && cd ~/esp/esp-idf && git checkout ${ESP_IDF_COMMIT} && ./install.sh esp32 esp32s3 && python ~/esp/esp-idf/tools/idf_tools.py install qemu-xtensa && virtualenv -p python3 /venv && source /venv/bin/activate && pip install --require-hashes -r /requirements.txt

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM blockstream/verde@sha256:391a4337c174f2eb47dc9f31d7e25d19a5f101bbc07fa8a296866c1ee52b8473
+FROM blockstream/verde@sha256:9171360dae48d3fba7c0f36ab5eb45e2529d178a603e89197cb08fb123ca619e
 
 RUN echo "source /root/esp/esp-idf/export.sh" >> /etc/bash.bashrc
 

Dockerfile.qemu

@@ -1,4 +1,4 @@
-FROM blockstream/verde@sha256:391a4337c174f2eb47dc9f31d7e25d19a5f101bbc07fa8a296866c1ee52b8473 as builder
+FROM blockstream/verde@sha256:9171360dae48d3fba7c0f36ab5eb45e2529d178a603e89197cb08fb123ca619e as builder
 
 ARG SDK_CONFIG=configs/sdkconfig_qemu_psram.defaults
 
@@ -12,15 +12,13 @@ RUN ./tools/fwprep.py build/jade.bin build
 ENV PATH=$PATH:/jade/main/qemu
 RUN make-flash-img.sh
 
-FROM debian:bookworm@sha256:321341744acb788e251ebd374aecc1a42d60ce65da7bd4ee9207ff6be6686a62
+FROM debian:bookworm-slim@sha256:12c396bd585df7ec21d5679bb6a83d4878bc4415ce926c9e5ea6426d23c60bdc
 
 RUN apt-get update -yqq && apt-get install -yqq libpixman-1-dev libglib2.0-dev libslirp-dev libncurses-dev
 
 COPY --from=builder /flash_image.bin /flash_image.bin
 COPY --from=builder /qemu_efuse.bin /qemu_efuse.bin
 
-COPY --from=builder /opt/bin/qemu-system-xtensa /opt/bin/qemu-system-xtensa
-COPY --from=builder /opt/share/qemu /opt/share/qemu
 COPY --from=builder /jade/main/qemu/ /jade/main/qemu/
 
 ENV PATH=$PATH:/jade/main/qemu

README.md

@@ -42,6 +42,7 @@ mkdir ~/esp
 cd ~/esp
 git clone -b v5.4 --recursive https://github.com/espressif/esp-idf.git
 cd ~/esp/esp-idf && git checkout 67c1de1eebe095d554d281952fde63c16ee2dca0 && ./install.sh --enable-gdbgui esp32 esp32s3
+python ~/esp/esp-idf/tools/idf_tools.py install qemu-xtensa
 ```
 
 Set up the environmental variables:

main/qemu/qemu_ci_flash.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -eo pipefail
 
+. $HOME/esp/esp-idf/export.sh
+
 # this is the minimum to run qemu
 # build/qemu-system-xtensa -nographic \
 #     -machine esp32 \
@@ -17,18 +19,17 @@ set -eo pipefail
 # -serial pty)
 # -serial mon:stdio)
 
-
 # if you want to put qemu in a state where it can be used with idf.py flash or esptool.py
 # -global driver=esp32.gpio,property=strap_mode,value=0x0f \
-
-/opt/bin/qemu-system-xtensa -nographic \
+qemu-system-xtensa -nographic \
     -machine esp32 \
     -m 4M \
     -drive file=/flash_image.bin,if=mtd,format=raw \
     -nic user,model=open_eth,id=lo0,hostfwd=tcp:0.0.0.0:30121-:30121 \
     -drive file=/qemu_efuse.bin,if=none,format=raw,id=efuse \
     -global driver=nvram.esp32.efuse,property=drive,value=efuse \
     -serial pty &
+
 sleep 4
 
 source /venv/bin/activate

main/qemu/qemu_gdb.sh

@@ -1,8 +1,12 @@
 #!/bin/bash
 
-pkill -f qemu-system-xtensa
+set -eo pipefail
 
-/opt/bin/qemu-system-xtensa -s -S -nographic \
+. $HOME/esp/esp-idf/export.sh
+
+pkill -f qemu-system-xtensa | true
+
+qemu-system-xtensa -s -S -nographic \
     -machine esp32 \
     -m 4M \
     -drive file=/flash_image.bin,if=mtd,format=raw \
@@ -19,4 +23,3 @@ xtensa-esp32-elf-gdb /jade/build/jade.elf \
     -ex "tb app_main" -ex "c" \
     -ex "b main.c:120"\
     -ex 'info b' -ex 'set print pretty on'
-

main/qemu/qemu_reboot.sh

@@ -1,8 +1,12 @@
 #!/bin/bash
 
-pkill -f qemu-system-xtensa
+set -eo pipefail
 
-/opt/bin/qemu-system-xtensa -nographic \
+. $HOME/esp/esp-idf/export.sh
+
+pkill -f qemu-system-xtensa || true
+
+qemu-system-xtensa -nographic \
     -machine esp32 \
     -m 4M \
     -drive file=/flash_image.bin,if=mtd,format=raw \

main/qemu/qemu_run.sh

@@ -1,8 +1,12 @@
 #!/bin/bash
+#
+set -eo pipefail
 
-pkill -f qemu-system-xtensa
+. $HOME/esp/esp-idf/export.sh
 
-/opt/bin/qemu-system-xtensa -nographic \
+pkill -f qemu-system-xtensa | true
+
+qemu-system-xtensa -nographic \
     -machine esp32 \
     -m 4M \
     -drive file=/flash_image.bin,if=mtd,format=raw \