Skip to content

Commit 7e9d98d

Browse files
lacraig2lacraig2
authored
optimizations and updates for CI/CD (#604)
* bump versions and use driver-opts * use rehosting arc registry * use insecure for registry mirror * log in after establishing that it's insecure * parsing * newline * cache mode max * cache to/from publish * formatting * again * registry updates * insecure * drop cache briefly * use login command * try again? * more * more * invalid * try more things? * cat out daemon.json * drop * sleep forever * fixup * dockerfile: allow registry setting * use registry in build-args * maybe? * fixup * use /proxy * update ca certificats * did we ever need these? * adjust * try? * try again * don't restart docker * setup buildx * fixup publish and build process * dockerfile: drop depencies txt files (caching is bad) * drop ./dependencies
1 parent 58dc4da commit 7e9d98d

File tree

5 files changed

+83
-120
lines changed

5 files changed

+83
-120
lines changed

‎.github/workflows/build.yaml‎

Lines changed: 42 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -37,46 +37,54 @@ jobs:
3737

3838
build_container:
3939
runs-on: rehosting-arc
40-
41-
# Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
42-
# (required)
4340
permissions:
4441
contents: write
4542
packages: write
4643
needs: lint
4744

4845
steps:
49-
- name: Log in to Docker Hub
50-
uses: docker/login-action@v3
51-
with:
52-
username: rehosting
53-
password: ${{secrets.DOCKERHUB_TOKEN}}
54-
55-
- name: Install dependencies and label git workspace safe
56-
run: |
57-
sudo apt-get update
58-
sudo apt-get -y install python3-pip git curl jq gzip tmux
59-
python3 -m pip install click
60-
git config --global --add safe.directory "$GITHUB_WORKSPACE"
61-
6246
- name: Checkout code
6347
uses: actions/checkout@v4
6448
with:
6549
fetch-depth: 0
50+
- name: Trust Harbor's self-signed certificate
51+
run: |
52+
echo "Fetching certificate from ${{ secrets.REHOSTING_ARC_REGISTRY }}"
53+
openssl s_client -showcerts -connect ${{ secrets.REHOSTING_ARC_REGISTRY }}:443 < /dev/null 2>/dev/null | openssl x509 -outform PEM | sudo tee /usr/local/share/ca-certificates/harbor.crt > /dev/null
54+
sudo update-ca-certificates
6655
6756
- name: Set up Docker Buildx
6857
uses: docker/setup-buildx-action@v3
58+
with:
59+
driver-opts: |
60+
image=moby/buildkit:master
61+
network=host
62+
buildkitd-config-inline: |
63+
[registry."${{ secrets.REHOSTING_ARC_REGISTRY }}"]
64+
insecure = true
65+
http = true
66+
67+
- name: Log in to Rehosting Arc Registry
68+
uses: docker/login-action@v3
69+
with:
70+
registry: ${{ secrets.REHOSTING_ARC_REGISTRY }}
71+
username: ${{ secrets.REHOSTING_ARC_REGISTRY_USER }}
72+
password: ${{ secrets.REHOSTING_ARC_REGISTRY_PASSWORD }}
6973

7074
- name: Build Docker image and push to Docker Hub
71-
uses: docker/build-push-action@v5
75+
uses: docker/build-push-action@v6
7276
with:
7377
context: .
7478
push: true
75-
tags: rehosting/penguin:${{ github.sha }}
79+
tags: ${{secrets.REHOSTING_ARC_REGISTRY}}/library/penguin:${{ github.sha }}
80+
build-args: |
81+
REGISTRY=${{ secrets.REHOSTING_ARC_REGISTRY }}/proxy
7682
cache-from: |
77-
type=registry,ref=rehosting/penguin:latest,mode=max
78-
type=registry,ref=rehosting/penguin:cache-PR-${{github.event.number}},mode=max
79-
cache-to: type=registry,ref=rehosting/penguin:cache-PR-${{ github.event.number }}
83+
type=registry,ref=${{secrets.REHOSTING_ARC_REGISTRY}}/library/penguin:cache,mode=max
84+
type=registry,ref=${{secrets.REHOSTING_ARC_REGISTRY}}/library/penguin:cache-PR-${{github.event.number}},mode=max
85+
cache-to: |
86+
type=registry,ref=${{secrets.REHOSTING_ARC_REGISTRY}}/library/penguin:cache,mode=max
87+
type=registry,ref=${{secrets.REHOSTING_ARC_REGISTRY}}/library/penguin:cache-PR-${{ github.event.number }}
8088
8189
run_tests:
8290
needs: build_container
@@ -98,18 +106,25 @@ jobs:
98106
with:
99107
fetch-depth: 0
100108

101-
- name: Log in to Docker Hub
109+
- name: Trust Harbor's self-signed certificate
110+
run: |
111+
echo "Fetching certificate from ${{ secrets.REHOSTING_ARC_REGISTRY }}"
112+
openssl s_client -showcerts -connect ${{ secrets.REHOSTING_ARC_REGISTRY }}:443 < /dev/null 2>/dev/null | openssl x509 -outform PEM | sudo tee /usr/local/share/ca-certificates/harbor.crt > /dev/null
113+
sudo update-ca-certificates
114+
115+
- name: Log in to Rehosting Arc Registry
102116
uses: docker/login-action@v3
103117
with:
104-
username: rehosting
105-
password: ${{secrets.DOCKERHUB_TOKEN}}
118+
registry: ${{ secrets.REHOSTING_ARC_REGISTRY }}
119+
username: ${{ secrets.REHOSTING_ARC_REGISTRY_USER }}
120+
password: ${{ secrets.REHOSTING_ARC_REGISTRY_PASSWORD }}
106121

107122

108123
# Locally tag as latest, just for testing
109-
- name: Pull the image from Dockerhub
124+
- name: Pull the image from Rehosting Arc Registry
110125
run: |
111-
docker pull rehosting/penguin:${{ github.sha }};
112-
docker tag rehosting/penguin:${{ github.sha }} rehosting/penguin:latest
126+
docker pull ${{secrets.REHOSTING_ARC_REGISTRY}}/library/penguin:${{ github.sha }};
127+
docker tag ${{secrets.REHOSTING_ARC_REGISTRY}}/library/penguin:${{ github.sha }} rehosting/penguin:latest
113128
114129
- name: Basic test for ${{ matrix.arch }}
115130
run: timeout 10m python3 $GITHUB_WORKSPACE/tests/unit_tests/basic_target/test.py --arch ${{ matrix.arch }}

‎.github/workflows/publish.yaml‎

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,19 @@ jobs:
2424
with:
2525
username: rehosting
2626
password: ${{secrets.DOCKERHUB_TOKEN}}
27+
28+
- name: Trust Harbor's self-signed certificate
29+
run: |
30+
echo "Fetching certificate from ${{ secrets.REHOSTING_ARC_REGISTRY }}"
31+
openssl s_client -showcerts -connect ${{ secrets.REHOSTING_ARC_REGISTRY }}:443 < /dev/null 2>/dev/null | openssl x509 -outform PEM | sudo tee /usr/local/share/ca-certificates/harbor.crt > /dev/null
32+
sudo update-ca-certificates
33+
34+
- name: Log in to Rehosting Arc Registry
35+
uses: docker/login-action@v3
36+
with:
37+
registry: ${{ secrets.REHOSTING_ARC_REGISTRY }}
38+
username: ${{ secrets.REHOSTING_ARC_REGISTRY_USER }}
39+
password: ${{ secrets.REHOSTING_ARC_REGISTRY_PASSWORD }}
2740

2841
- name: Install dependencies and label git workspace safe
2942
run: |
@@ -35,26 +48,39 @@ jobs:
3548
uses: actions/checkout@v4
3649
with:
3750
fetch-depth: 0
38-
51+
52+
- name: Set up Docker Buildx
53+
uses: docker/setup-buildx-action@v3
54+
with:
55+
driver-opts: |
56+
image=moby/buildkit:master
57+
network=host
58+
buildkitd-config-inline: |
59+
[registry."${{ secrets.REHOSTING_ARC_REGISTRY }}"]
60+
insecure = true
61+
http = true
62+
3963
- name: Build Docker image and push to Dockerhub
40-
uses: docker/build-push-action@v6.3.0
64+
uses: docker/build-push-action@v6.18.0
4165
with:
4266
context: .
4367
push: true
44-
cache-from: type=registry,ref=rehosting/penguin:latest
45-
cache-to: type=inline
68+
cache-from: |
69+
type=registry,ref=${{secrets.REHOSTING_ARC_REGISTRY}}/library/penguin:cache,mode=max
70+
cache-to: |
71+
type=registry,ref=${{secrets.REHOSTING_ARC_REGISTRY}}/library/penguin:cache,mode=max
4672
tags: rehosting/penguin:${{ github.sha }},rehosting/penguin:${{ steps.version.outputs.v-version }},rehosting/penguin:latest
4773
build-args: |
4874
OVERRIDE_VERSION=${{ steps.version.outputs.v-version }}
4975
5076
- name: Create release
5177
id: create_release
52-
uses: softprops/action-gh-release@v2.0.8
78+
uses: softprops/action-gh-release@v2.3.2
5379
env:
5480
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
5581
with:
5682
tag_name: ${{ steps.version.outputs.v-version }}
57-
release_name: Release ${{ steps.version.outputs.v-version }} ${{ github.ref }}
83+
name: Release ${{ steps.version.outputs.v-version }} ${{ github.ref }}
5884
body: |
5985
Release ${{ steps.version.outputs.v-version }} @${{ github.ref }}
6086
draft: false

‎Dockerfile‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# versions of the various dependencies.
2-
ARG BASE_IMAGE="ubuntu:22.04"
2+
ARG REGISTRY="docker.io"
3+
ARG BASE_IMAGE="${REGISTRY}/ubuntu:22.04"
34
ARG DOWNLOAD_TOKEN="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"
45
ARG VPN_VERSION="1.0.24"
56
ARG BUSYBOX_VERSION="0.0.15"
@@ -19,7 +20,7 @@ ARG PANDA_VERSION="pandav0.0.37"
1920
ARG PANDANG_VERSION="0.0.26"
2021
ARG RIPGREP_VERSION="14.1.1"
2122

22-
FROM rust:1.86 as rust_builder
23+
FROM ${REGISTRY}/rust:1.86 AS rust_builder
2324
RUN git clone --depth 1 -q https://github.com/rust-vmm/vhost-device/ /root/vhost-device
2425
ARG VHOST_DEVICE_VERSION
2526
ENV PATH="/root/.cargo/bin:$PATH"
@@ -167,7 +168,7 @@ COPY ./src/resources/ltrace_nvram.conf /tmp/ltrace/lib_inject.so.conf
167168

168169

169170
#### CROSS BUILDER: Build send_hypercall ###
170-
FROM ghcr.io/rehosting/embedded-toolchains:latest AS cross_builder
171+
FROM ${REGISTRY}/rehosting/embedded-toolchains:latest AS cross_builder
171172
COPY ./guest-utils/native/ /source
172173
WORKDIR /source
173174
RUN wget -q https://raw.githubusercontent.com/panda-re/libhc/main/hypercall.h
@@ -303,8 +304,7 @@ RUN if [ ! -z "${OVERRIDE_VERSION}" ]; then \
303304
FROM $BASE_IMAGE AS fw2tar_dep_builder
304305
ENV DEBIAN_FRONTEND=noninteractive
305306

306-
COPY ./dependencies/fw2tar.txt /tmp/fw2tar.txt
307-
RUN apt-get update && apt-get install -y -q git $(cat /tmp/fw2tar.txt)
307+
RUN apt-get update && apt-get install -y -q git android-sdk-libsparse-utils arj automake build-essential bzip2 cabextract clang cpio cramfsswap curl default-jdk e2fsprogs fakeroot gcc git gzip lhasa libarchive-dev libfontconfig1-dev libacl1-dev libcap-dev liblzma-dev liblzo2-dev liblz4-dev libbz2-dev libssl-dev libmagic1 locales lz4 lziprecover lzop mtd-utils openssh-client p7zip p7zip-full python3 python3-pip qtbase5-dev sleuthkit squashfs-tools srecord tar unar unrar unrar-free unyaffs unzip wget xz-utils zlib1g-dev zstd
308308

309309
ARG DOWNLOAD_TOKEN
310310
ARG FW2TAR_TAG
@@ -345,7 +345,6 @@ COPY --from=downloader /tmp/pandare-plugins.deb /tmp/
345345
COPY --from=downloader /tmp/glow.deb /tmp/
346346
COPY --from=downloader /tmp/gum.deb /tmp/
347347
COPY --from=downloader /tmp/ripgrep.deb /tmp/
348-
COPY ./dependencies/* /tmp
349348

350349
# We need pycparser>=2.21 for angr. If we try this later with the other pip commands,
351350
# we'll fail because we get a distutils distribution of pycparser 2.19 that we can't
@@ -372,8 +371,10 @@ RUN wget https://apt.llvm.org/llvm.sh && \
372371
chmod +x llvm.sh && \
373372
./llvm.sh 20
374373

375-
# Install apt dependencies - largely for binwalk, some for penguin, some for fw2tar
376-
RUN apt-get update && apt-get install -q -y $(cat /tmp/penguin.txt) $(cat /tmp/fw2tar.txt) && \
374+
# Install apt dependencies - first line for penguin - second for fw2tar
375+
RUN apt-get update && apt-get install -q -y \
376+
fakeroot genext2fs graphviz graphviz-dev libarchive13 libgcc-s1 liblinear4 liblua5.3-0 libpcap0.8 libpcre3 libssh2-1 libssl3 libstdc++6 libxml2 lua-lpeg nmap python3 python3-lxml python3-venv sudo telnet vim wget zlib1g pigz clang-20 lldb-20 lld-20 \
377+
android-sdk-libsparse-utils arj automake build-essential bzip2 cabextract clang cpio cramfsswap curl default-jdk e2fsprogs fakeroot gcc git gzip lhasa libarchive-dev libfontconfig1-dev libacl1-dev libcap-dev liblzma-dev liblzo2-dev liblz4-dev libbz2-dev libssl-dev libmagic1 locales lz4 lziprecover lzop mtd-utils openssh-client p7zip p7zip-full python3 python3-pip qtbase5-dev sleuthkit squashfs-tools srecord tar unar unrar unrar-free unyaffs unzip wget xz-utils zlib1g-dev zstd && \
377378
apt install -yy -f /tmp/pandare.deb -f /tmp/pandare-plugins.deb \
378379
-f /tmp/glow.deb -f /tmp/gum.deb -f /tmp/ripgrep.deb && \
379380
rm -rf /var/lib/apt/lists/* /tmp/*.deb

‎dependencies/fw2tar.txt‎

Lines changed: 0 additions & 51 deletions
This file was deleted.

‎dependencies/penguin.txt‎

Lines changed: 0 additions & 28 deletions
This file was deleted.

0 commit comments

Comments
 (0)