Skip to content

Commit 6fce200

Browse files
veblushveblush
authored
[CI] Install qemu (#3201)
- Improved ci/Dockerfile.micro further by - Installing QEMU from packages - Optimizing the docker definition by combining multiple layers into single layer to reduce the footprint. BUG=Clean-up
1 parent 598c48e commit 6fce200

File tree

6 files changed

+71
-86
lines changed

6 files changed

+71
-86
lines changed

‎.github/workflows/check_tflite_files.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ jobs:
3434
echo "${PR_FILES}" >> tmp_pull_request_files.txt
3535
rm -rf .git
3636
echo ${{ secrets.tflm-bot-token }} | docker login ghcr.io -u tflm-bot --password-stdin
37-
docker run --rm -v `pwd`:/tflite-micro -w /tflite-micro ghcr.io/tflm-bot/tflm-ci:0.6.3 python3 ci/check_tflite_files.py tmp_pull_request_files.txt
37+
docker run --rm -v `pwd`:/tflite-micro -w /tflite-micro ghcr.io/tflm-bot/tflm-ci:0.6.4 python3 ci/check_tflite_files.py tmp_pull_request_files.txt
3838
TFLITE_FILE_TEST_STATUS=$?
3939
rm -f tmp_pull_request_files.txt
4040
exit ${TFLITE_FILE_TEST_STATUS}

‎.github/workflows/ci.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -259,7 +259,7 @@ jobs:
259259
with:
260260
ref: ${{ inputs.trigger-sha }}
261261
- name: Test
262-
uses: docker://ghcr.io/tflm-bot/tflm-ci:0.6.3
262+
uses: docker://ghcr.io/tflm-bot/tflm-ci:0.6.4
263263
with:
264264
args: /bin/sh -c tensorflow/lite/micro/tools/ci_build/test_cortex_m_qemu.sh tflite-micro/
265265

@@ -275,7 +275,7 @@ jobs:
275275
with:
276276
ref: ${{ inputs.trigger-sha }}
277277
- name: Check
278-
uses: docker://ghcr.io/tflm-bot/tflm-ci:0.6.3
278+
uses: docker://ghcr.io/tflm-bot/tflm-ci:0.6.4
279279
with:
280280
args: /bin/sh -c "git config --global --add safe.directory /github/workspace && tensorflow/lite/micro/tools/ci_build/test_code_style.sh"
281281

‎.github/workflows/riscv.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ jobs:
2424
python-version: '3.10'
2525
- uses: actions/checkout@v3
2626
- name: Test
27-
uses: docker://ghcr.io/tflm-bot/tflm-ci:0.6.3
27+
uses: docker://ghcr.io/tflm-bot/tflm-ci:0.6.4
2828
with:
2929
args: /bin/sh -c tensorflow/lite/micro/tools/ci_build/test_riscv.sh
3030

‎.github/workflows/riscv_postmerge.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,6 @@ jobs:
3232
with:
3333
ref: ${{ inputs.trigger-sha }}
3434
- name: Test
35-
uses: docker://ghcr.io/tflm-bot/tflm-ci:0.6.3
35+
uses: docker://ghcr.io/tflm-bot/tflm-ci:0.6.4
3636
with:
3737
args: /bin/sh -c tensorflow/lite/micro/tools/ci_build/test_riscv.sh

‎ci/Dockerfile.micro‎

Lines changed: 66 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -3,92 +3,80 @@
33
# This container includes all dependencies for TFLM CI checks and is
44
# recommended for local debugging to ensure a consistent environment.
55
#
6-
# * Pull a prebuilt image:
7-
# See all versions: https://github.com/users/TFLM-bot/packages/container/tflm-ci/versions
6+
# === Usage ===
7+
#
8+
# --- Pull a prebuilt image ---
9+
# See all versions: https://github.com/users/TFLM-bot/packages/container/tflm-ci/versions
810
# docker pull ghcr.io/tflm-bot/tflm-ci:<version>
911
#
10-
# * Build locally:
11-
# (Run from the root of the TFLM repository)
12+
# --- Build locally ---
13+
# (Run from the root of the TFLM repository)
1214
# docker build -f ci/Dockerfile.micro -t tflm-ci .
1315
#
14-
# * Build and upload to GHCR.IO:
15-
# (Run from the root of the TFLM repository)
16-
# (Update the TFLM_CI_VERSION variable below to your target version)
17-
# export TFLM_CI_VERSION=0.6.3
16+
# --- Build and upload to GHCR.IO ---
17+
# (Run from the root of the TFLM repository)
18+
# export TFLM_CI_VERSION=0.6.4
1819
# docker build -f ci/Dockerfile.micro -t ghcr.io/tflm-bot/tflm-ci:$TFLM_CI_VERSION .
1920
# docker push ghcr.io/tflm-bot/tflm-ci:$TFLM_CI_VERSION
20-
# (And you need to update github workflows to use this version.)
2121
#
22-
# * Run locally for debugging:
23-
# (Mounts local repo and opens a shell)
24-
# (Run from the root of the TFLM repository)
22+
# --- Run locally for debugging ---
23+
# (Mounts local repo and opens a shell)
24+
# (Run from the root of the TFLM repository)
2525
# docker run -it --rm -v $(pwd):/opt/tflm -w /opt/tflm tflm-ci /bin/bash
26+
#
2627

27-
# Use a prebuilt Python image instead of base Ubuntu to speed up the build process,
28-
# since it has all the build dependencies we need for Micro and downloads much faster
29-
# than the install process.
30-
31-
# Using a multistage build so that the build tools required for stage 1 don't make the
32-
# CI container unnecessarily large.
33-
FROM python:3.10-bookworm AS qemu_builder
34-
RUN apt-get update
35-
36-
RUN apt-get install -y ninja-build wget xz-utils
37-
RUN apt-get install -y libglib2.0-dev
38-
RUN apt-get install -y build-essential libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev
39-
40-
COPY ci/install_qemu.sh ./
41-
# Installs all built files into /qemu_install rather than /usr/local default.
42-
RUN ./install_qemu.sh /tmp/qemu_install
43-
44-
# This stage is the final CI container.
45-
FROM python:3.10-bookworm AS tflm-ci
46-
47-
RUN apt-get update
48-
49-
RUN apt-get install -y zip xxd sudo
50-
51-
RUN apt install -y lsb-release wget software-properties-common gnupg
52-
RUN wget https://apt.llvm.org/llvm.sh
53-
RUN chmod +x llvm.sh
54-
RUN ./llvm.sh 21
55-
RUN ln -s /usr/bin/clang-21 /usr/bin/clang
56-
RUN ln -s /usr/bin/clang++-21 /usr/bin/clang++
57-
58-
RUN apt-get install clang-format-21
59-
RUN ln -s /usr/bin/clang-format-21 /usr/bin/clang-format
60-
61-
# Needed when using the Dockerfile locally.
62-
RUN git config --global --add safe.directory /opt/tflm
63-
64-
# Needed when the docker container is used with GitHub actions.
65-
RUN git config --global --add safe.directory /github/workspace
66-
67-
# Install yapf to check for Python formatting as part of the TFLM continuous
68-
# integration.
69-
RUN pip install yapf==0.40.2
70-
71-
# Pillow was added first for the C array generation as a result of the following
72-
# PRs:
73-
# https://github.com/tensorflow/tflite-micro/pull/337
74-
# https://github.com/tensorflow/tflite-micro/pull/410
75-
RUN pip install Pillow
76-
77-
# necessary bits for create_size_log scripts
78-
RUN pip install pandas
79-
RUN pip install matplotlib
80-
RUN pip install six
81-
82-
# Install Renode test dependencies
83-
RUN pip install pyyaml requests psutil robotframework==4.0.1
84-
85-
# Install QEMU from build container qemu_builder into tflm-ci container.
86-
# We're using a two stage build to keep the CI container smaller.
87-
WORKDIR /usr/local
88-
# Merge built files into /usr/local so that the path is already setup.
89-
COPY --from=qemu_builder /tmp/qemu_install/. .
90-
28+
# Use a prebuilt Python image to speed up the build process.
29+
FROM python:3.10-bookworm
30+
31+
# Install all required system dependencies
32+
RUN apt-get update && apt-get install -y \
33+
gnupg \
34+
lsb-release \
35+
qemu-system-arm \
36+
software-properties-common \
37+
sudo \
38+
wget \
39+
xxd \
40+
zip \
41+
&& apt-get clean \
42+
&& rm -rf /var/lib/apt/lists/*
43+
44+
# Install LLVM/Clang and create symlinks
45+
RUN wget https://apt.llvm.org/llvm.sh \
46+
&& chmod +x llvm.sh \
47+
&& ./llvm.sh 21 \
48+
&& rm llvm.sh \
49+
&& apt-get update && apt-get install -y clang-21 clang++-21 clang-format-21 \
50+
&& ln -s /usr/bin/clang-21 /usr/bin/clang \
51+
&& ln -s /usr/bin/clang++-21 /usr/bin/clang++ \
52+
&& ln -s /usr/bin/clang-format-21 /usr/bin/clang-format \
53+
&& apt-get clean \
54+
&& rm -rf /var/lib/apt/lists/*
55+
56+
# Install all Python dependencies
57+
# - yapf to check for Python formatting
58+
# - Pillow was added first for the C array generation
59+
# - pandas, matplotlib, and six for create_size_log scripts
60+
# - pyyaml, requests, psutil, and robotframework for Renode test
61+
RUN pip install --no-cache-dir --upgrade pip && \
62+
pip install --no-cache-dir \
63+
matplotlib \
64+
pandas \
65+
Pillow \
66+
psutil \
67+
pyyaml \
68+
requests \
69+
robotframework==4.0.1 \
70+
six \
71+
yapf==0.40.2
72+
73+
# Copy and run install scripts, then clean them up
9174
WORKDIR /
9275
COPY ci/*.sh /install/
93-
RUN /install/install_bazelisk.sh
94-
RUN /install/install_buildifier.sh
76+
RUN /install/install_bazelisk.sh && \
77+
/install/install_buildifier.sh && \
78+
rm -rf /install
79+
80+
# Configure git safe directories
81+
RUN git config --global --add safe.directory /opt/tflm && \
82+
git config --global --add safe.directory /github/workspace

‎tensorflow/lite/micro/docs/qemu.md‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,6 @@ Our test scripts assume that the non static `user` mode installation of QEMU is
1313
available in the PATH. For example, if using QEMU for ARM testing, please make
1414
sure `qemu-arm` is installed and available to the test scripts.
1515

16-
You can use `ci/install_qemu.sh` to download, build and install the version of
17-
qemu that is used as part of the CI.
18-
1916
# Software Emulation with QEMU
2017
TensorFlow Lite Micro makes use of [QEMU](https://qemu.org) to
2118
for testing cross compiled tests.

0 commit comments

Comments
 (0)