Skip to content

Commit 3358b53

Browse files
darthShadowdarthShadow
committed
Added parallel docker builds and caching for go build in the container
Signed-off-by: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com>
1 parent 5a4026c commit 3358b53

File tree

4 files changed

+329
-166
lines changed

4 files changed

+329
-166
lines changed

‎.github/workflows/build_publish_beta_docker_image.yml‎

Lines changed: 0 additions & 77 deletions
This file was deleted.

‎.github/workflows/build_publish_docker_image.yml‎

Lines changed: 284 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,284 @@
1+
---
2+
# Github Actions release for rclone
3+
# -*- compile-command: "yamllint -f build_publish_docker_image.yml" -*-
4+
5+
name: Build & Push Docker Images
6+
7+
# Trigger the workflow on push or pull request
8+
on:
9+
push:
10+
branches:
11+
- '**'
12+
tags:
13+
- '**'
14+
pull_request:
15+
workflow_dispatch:
16+
inputs:
17+
manual:
18+
description: Manual run (bypass default conditions)
19+
type: boolean
20+
default: true
21+
22+
jobs:
23+
build-image:
24+
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
25+
timeout-minutes: 60
26+
strategy:
27+
fail-fast: false
28+
matrix:
29+
include:
30+
- platform: linux/amd64
31+
runs-on: ubuntu-latest
32+
- platform: linux/386
33+
runs-on: ubuntu-latest
34+
- platform: linux/arm64
35+
runs-on: ubuntu-latest
36+
- platform: linux/arm/v7
37+
runs-on: ubuntu-latest
38+
- platform: linux/arm/v6
39+
runs-on: ubuntu-latest
40+
41+
name: Build Docker Image for ${{ matrix.platform }}
42+
runs-on: ${{ matrix.runs-on }}
43+
44+
steps:
45+
- name: Free Space
46+
shell: bash
47+
run: |
48+
df -h .
49+
# Remove android SDK
50+
sudo rm -rf /usr/local/lib/android || true
51+
# Remove .net runtime
52+
sudo rm -rf /usr/share/dotnet || true
53+
df -h .
54+
55+
- name: Checkout Repository
56+
uses: actions/checkout@v4
57+
with:
58+
fetch-depth: 0
59+
60+
- name: Set REPO_NAME Variable
61+
run: |
62+
echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV}
63+
64+
- name: Set PLATFORM Variable
65+
run: |
66+
platform=${{ matrix.platform }}
67+
echo "PLATFORM=${platform//\//-}" >> $GITHUB_ENV
68+
69+
- name: Set CACHE_NAME Variable
70+
shell: python
71+
run: |
72+
import os, re
73+
74+
def slugify(input_string, max_length=63):
75+
slug = input_string.lower()
76+
slug = re.sub(r'[^a-z0-9 -]', ' ', slug)
77+
slug = slug.strip()
78+
slug = re.sub(r'\s+', '-', slug)
79+
slug = re.sub(r'-+', '-', slug)
80+
slug = slug[:max_length]
81+
slug = re.sub(r'[-]+$', '', slug)
82+
return slug
83+
84+
ref_name_slug = "cache"
85+
86+
if os.environ.get("GITHUB_REF_NAME") and os.environ['GITHUB_EVENT_NAME'] == "pull_request":
87+
ref_name_slug += "-pr-" + slugify(os.environ['GITHUB_REF_NAME'])
88+
89+
with open(os.environ['GITHUB_ENV'], 'a') as env:
90+
env.write(f"CACHE_NAME={ref_name_slug}\n")
91+
92+
- name: Extract Metadata (tags, labels) for Docker
93+
id: meta
94+
uses: docker/metadata-action@v5
95+
env:
96+
DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,manifest-descriptor # Important for digest annotation (used by Github packages)
97+
with:
98+
images: |
99+
ghcr.io/${{ env.REPO_NAME }}
100+
labels: |
101+
org.opencontainers.image.url=https://github.com/rclone/rclone/pkgs/container/rclone
102+
org.opencontainers.image.vendor=${{ github.repository_owner }}
103+
org.opencontainers.image.authors=rclone <https://github.com/rclone>
104+
org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
105+
org.opencontainers.image.revision=${{ github.sha }}
106+
tags: |
107+
type=sha
108+
type=ref,event=pr
109+
type=ref,event=branch
110+
type=semver,pattern={{version}}
111+
type=semver,pattern={{major}}
112+
type=semver,pattern={{major}}.{{minor}}
113+
type=raw,value=beta,enable={{is_default_branch}}
114+
115+
- name: Setup QEMU
116+
uses: docker/setup-qemu-action@v3
117+
118+
- name: Set up Docker Buildx
119+
uses: docker/setup-buildx-action@v3
120+
121+
- name: Load Go Build Cache for Docker
122+
id: go-cache
123+
uses: actions/cache@v4
124+
with:
125+
# Cache only the go builds, the module download is cached via the docker layer caching
126+
path: |
127+
go-build-cache
128+
key: ${{ runner.os }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }}
129+
restore-keys: |
130+
${{ runner.os }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}
131+
132+
- name: Inject Go Build Cache into Docker
133+
uses: reproducible-containers/buildkit-cache-dance@v3
134+
with:
135+
cache-map: |
136+
{
137+
"go-build-cache": "/root/.cache/go-build"
138+
}
139+
skip-extraction: ${{ steps.cache.outputs.go-cache-hit }}
140+
141+
- name: Login to GitHub Container Registry
142+
uses: docker/login-action@v3
143+
with:
144+
registry: ghcr.io
145+
# This is the user that triggered the Workflow. In this case, it will
146+
# either be the user whom created the Release or manually triggered
147+
# the workflow_dispatch.
148+
username: ${{ github.actor }}
149+
password: ${{ secrets.GITHUB_TOKEN }}
150+
151+
- name: Build and Publish Image Digest
152+
id: build
153+
uses: docker/build-push-action@v6
154+
with:
155+
file: Dockerfile
156+
context: .
157+
provenance: false
158+
# don't specify 'tags' here (error "get can't push tagged ref by digest")
159+
# tags: ${{ steps.meta.outputs.tags }}
160+
labels: ${{ steps.meta.outputs.labels }}
161+
annotations: ${{ steps.meta.outputs.annotations }}
162+
platforms: ${{ matrix.platform }}
163+
outputs: |
164+
type=image,name=ghcr.io/${{ env.REPO_NAME }},push-by-digest=true,name-canonical=true,push=true
165+
cache-from: |
166+
type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}
167+
cache-to: |
168+
type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.CACHE_NAME }}-${{ env.PLATFORM }},image-manifest=true,mode=max,compression=zstd
169+
170+
- name: Export Image Digest
171+
run: |
172+
mkdir -p /tmp/digests
173+
digest="${{ steps.build.outputs.digest }}"
174+
touch "/tmp/digests/${digest#sha256:}"
175+
176+
- name: Upload Image Digest
177+
uses: actions/upload-artifact@v4
178+
with:
179+
name: digests-${{ env.PLATFORM }}
180+
path: /tmp/digests/*
181+
retention-days: 1
182+
if-no-files-found: error
183+
184+
merge-image:
185+
runs-on: ubuntu-latest
186+
needs:
187+
- build-image
188+
189+
steps:
190+
- name: Download Image Digests
191+
uses: actions/download-artifact@v4
192+
with:
193+
path: /tmp/digests
194+
pattern: digests-*
195+
merge-multiple: true
196+
197+
- name: Set REPO_NAME Variable
198+
run: |
199+
echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV}
200+
201+
- name: Extract Metadata (tags, labels) for Docker
202+
id: meta
203+
uses: docker/metadata-action@v5
204+
env:
205+
DOCKER_METADATA_ANNOTATIONS_LEVELS: index
206+
with:
207+
images: |
208+
${{ env.REPO_NAME }}
209+
ghcr.io/${{ env.REPO_NAME }}
210+
labels: |
211+
org.opencontainers.image.url=https://github.com/rclone/rclone/pkgs/container/rclone
212+
org.opencontainers.image.vendor=${{ github.repository_owner }}
213+
org.opencontainers.image.authors=rclone <https://github.com/rclone>
214+
org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
215+
org.opencontainers.image.revision=${{ github.sha }}
216+
tags: |
217+
type=sha
218+
type=ref,event=pr
219+
type=ref,event=branch
220+
type=semver,pattern={{version}}
221+
type=semver,pattern={{major}}
222+
type=semver,pattern={{major}}.{{minor}}
223+
type=raw,value=beta,enable={{is_default_branch}}
224+
225+
- name: Extract Tags
226+
shell: python
227+
run: |
228+
import json, os
229+
230+
metadata_json = os.environ['DOCKER_METADATA_OUTPUT_JSON']
231+
metadata = json.loads(metadata_json)
232+
233+
tags = [f"--tag '{tag}'" for tag in metadata["tags"]]
234+
tags_string = " ".join(tags)
235+
236+
with open(os.environ['GITHUB_ENV'], 'a') as env:
237+
env.write(f"TAGS={tags_string}\n")
238+
239+
- name: Extract Annotations
240+
shell: python
241+
run: |
242+
import json, os
243+
244+
metadata_json = os.environ['DOCKER_METADATA_OUTPUT_JSON']
245+
metadata = json.loads(metadata_json)
246+
247+
annotations = [f"--annotation '{annotation}'" for annotation in metadata["annotations"]]
248+
annotations_string = " ".join(annotations)
249+
250+
with open(os.environ['GITHUB_ENV'], 'a') as env:
251+
env.write(f"ANNOTATIONS={annotations_string}\n")
252+
253+
- name: Set up Docker Buildx
254+
uses: docker/setup-buildx-action@v3
255+
256+
- name: Login to Docker Hub
257+
uses: docker/login-action@v3
258+
with:
259+
username: ${{ secrets.DOCKERHUB_USERNAME }}
260+
password: ${{ secrets.DOCKERHUB_TOKEN }}
261+
262+
- name: Login to GitHub Container Registry
263+
uses: docker/login-action@v3
264+
with:
265+
registry: ghcr.io
266+
# This is the user that triggered the Workflow. In this case, it will
267+
# either be the user whom created the Release or manually triggered
268+
# the workflow_dispatch.
269+
username: ${{ github.actor }}
270+
password: ${{ secrets.GITHUB_TOKEN }}
271+
272+
- name: Create & Push Manifest List
273+
working-directory: /tmp/digests
274+
run: |
275+
docker buildx imagetools create \
276+
${{ env.TAGS }} \
277+
${{ env.ANNOTATIONS }} \
278+
$(printf 'ghcr.io/${{ env.REPO_NAME }}@sha256:%s ' *)
279+
280+
- name: Inspect and Run Multi-Platform Image
281+
run: |
282+
docker buildx imagetools inspect --raw ${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }}
283+
docker buildx imagetools inspect --raw ghcr.io/${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }}
284+
docker run --rm ghcr.io/${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }} version

0 commit comments

Comments
 (0)