Skip to content

Commit c0515a5

Browse files
darthShadowncw
darthShadow
authored and
ncw
committed
Added parallel docker builds and caching for go build in the container
Signed-off-by: Anagh Kumar Baranwal <[email protected]>
1 parent dc9c872 commit c0515a5

File tree

5 files changed

+339
-168
lines changed

5 files changed

+339
-168
lines changed

‎.github/workflows/build_publish_beta_docker_image.yml‎

Lines changed: 0 additions & 77 deletions
This file was deleted.

‎.github/workflows/build_publish_docker_image.yml‎

Lines changed: 294 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,294 @@
1+
---
2+
# Github Actions release for rclone
3+
# -*- compile-command: "yamllint -f build_publish_docker_image.yml" -*-
4+
5+
name: Build & Push Docker Images
6+
7+
# Trigger the workflow on push or pull request
8+
on:
9+
push:
10+
branches:
11+
- '**'
12+
tags:
13+
- '**'
14+
pull_request:
15+
workflow_dispatch:
16+
inputs:
17+
manual:
18+
description: Manual run (bypass default conditions)
19+
type: boolean
20+
default: true
21+
22+
jobs:
23+
build-image:
24+
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
25+
timeout-minutes: 60
26+
strategy:
27+
fail-fast: false
28+
matrix:
29+
include:
30+
- platform: linux/amd64
31+
runs-on: ubuntu-24.04
32+
- platform: linux/386
33+
runs-on: ubuntu-24.04
34+
- platform: linux/arm64
35+
runs-on: ubuntu-24.04-arm
36+
- platform: linux/arm/v7
37+
runs-on: ubuntu-24.04-arm
38+
- platform: linux/arm/v6
39+
runs-on: ubuntu-24.04-arm
40+
41+
name: Build Docker Image for ${{ matrix.platform }}
42+
runs-on: ${{ matrix.runs-on }}
43+
44+
steps:
45+
- name: Free Space
46+
shell: bash
47+
run: |
48+
df -h .
49+
# Remove android SDK
50+
sudo rm -rf /usr/local/lib/android || true
51+
# Remove .net runtime
52+
sudo rm -rf /usr/share/dotnet || true
53+
df -h .
54+
55+
- name: Checkout Repository
56+
uses: actions/checkout@v4
57+
with:
58+
fetch-depth: 0
59+
60+
- name: Set REPO_NAME Variable
61+
run: |
62+
echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV}
63+
64+
- name: Set PLATFORM Variable
65+
run: |
66+
platform=${{ matrix.platform }}
67+
echo "PLATFORM=${platform//\//-}" >> $GITHUB_ENV
68+
69+
- name: Set CACHE_NAME Variable
70+
shell: python
71+
run: |
72+
import os, re
73+
74+
def slugify(input_string, max_length=63):
75+
slug = input_string.lower()
76+
slug = re.sub(r'[^a-z0-9 -]', ' ', slug)
77+
slug = slug.strip()
78+
slug = re.sub(r'\s+', '-', slug)
79+
slug = re.sub(r'-+', '-', slug)
80+
slug = slug[:max_length]
81+
slug = re.sub(r'[-]+$', '', slug)
82+
return slug
83+
84+
ref_name_slug = "cache"
85+
86+
if os.environ.get("GITHUB_REF_NAME") and os.environ['GITHUB_EVENT_NAME'] == "pull_request":
87+
ref_name_slug += "-pr-" + slugify(os.environ['GITHUB_REF_NAME'])
88+
89+
with open(os.environ['GITHUB_ENV'], 'a') as env:
90+
env.write(f"CACHE_NAME={ref_name_slug}\n")
91+
92+
- name: Get ImageOS
93+
# There's no way around this, because "ImageOS" is only available to
94+
# processes, but the setup-go action uses it in its key.
95+
id: imageos
96+
uses: actions/github-script@v7
97+
with:
98+
result-encoding: string
99+
script: |
100+
return process.env.ImageOS
101+
102+
- name: Extract Metadata (tags, labels) for Docker
103+
id: meta
104+
uses: docker/metadata-action@v5
105+
env:
106+
DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,manifest-descriptor # Important for digest annotation (used by Github packages)
107+
with:
108+
images: |
109+
ghcr.io/${{ env.REPO_NAME }}
110+
labels: |
111+
org.opencontainers.image.url=https://github.com/rclone/rclone/pkgs/container/rclone
112+
org.opencontainers.image.vendor=${{ github.repository_owner }}
113+
org.opencontainers.image.authors=rclone <https://github.com/rclone>
114+
org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
115+
org.opencontainers.image.revision=${{ github.sha }}
116+
tags: |
117+
type=sha
118+
type=ref,event=pr
119+
type=ref,event=branch
120+
type=semver,pattern={{version}}
121+
type=semver,pattern={{major}}
122+
type=semver,pattern={{major}}.{{minor}}
123+
type=raw,value=beta,enable={{is_default_branch}}
124+
125+
- name: Setup QEMU
126+
uses: docker/setup-qemu-action@v3
127+
128+
- name: Set up Docker Buildx
129+
uses: docker/setup-buildx-action@v3
130+
131+
- name: Load Go Build Cache for Docker
132+
id: go-cache
133+
uses: actions/cache@v4
134+
with:
135+
key: ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }}
136+
restore-keys: |
137+
${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}
138+
# Cache only the go builds, the module download is cached via the docker layer caching
139+
path: |
140+
go-build-cache
141+
142+
- name: Inject Go Build Cache into Docker
143+
uses: reproducible-containers/buildkit-cache-dance@v3
144+
with:
145+
cache-map: |
146+
{
147+
"go-build-cache": "/root/.cache/go-build"
148+
}
149+
skip-extraction: ${{ steps.go-cache.outputs.cache-hit }}
150+
151+
- name: Login to GitHub Container Registry
152+
uses: docker/login-action@v3
153+
with:
154+
registry: ghcr.io
155+
# This is the user that triggered the Workflow. In this case, it will
156+
# either be the user whom created the Release or manually triggered
157+
# the workflow_dispatch.
158+
username: ${{ github.actor }}
159+
password: ${{ secrets.GITHUB_TOKEN }}
160+
161+
- name: Build and Publish Image Digest
162+
id: build
163+
uses: docker/build-push-action@v6
164+
with:
165+
file: Dockerfile
166+
context: .
167+
provenance: false
168+
# don't specify 'tags' here (error "get can't push tagged ref by digest")
169+
# tags: ${{ steps.meta.outputs.tags }}
170+
labels: ${{ steps.meta.outputs.labels }}
171+
annotations: ${{ steps.meta.outputs.annotations }}
172+
platforms: ${{ matrix.platform }}
173+
outputs: |
174+
type=image,name=ghcr.io/${{ env.REPO_NAME }},push-by-digest=true,name-canonical=true,push=true
175+
cache-from: |
176+
type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}
177+
cache-to: |
178+
type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.CACHE_NAME }}-${{ env.PLATFORM }},image-manifest=true,mode=max,compression=zstd
179+
180+
- name: Export Image Digest
181+
run: |
182+
mkdir -p /tmp/digests
183+
digest="${{ steps.build.outputs.digest }}"
184+
touch "/tmp/digests/${digest#sha256:}"
185+
186+
- name: Upload Image Digest
187+
uses: actions/upload-artifact@v4
188+
with:
189+
name: digests-${{ env.PLATFORM }}
190+
path: /tmp/digests/*
191+
retention-days: 1
192+
if-no-files-found: error
193+
194+
merge-image:
195+
runs-on: ubuntu-24.04
196+
needs:
197+
- build-image
198+
199+
steps:
200+
- name: Download Image Digests
201+
uses: actions/download-artifact@v4
202+
with:
203+
path: /tmp/digests
204+
pattern: digests-*
205+
merge-multiple: true
206+
207+
- name: Set REPO_NAME Variable
208+
run: |
209+
echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV}
210+
211+
- name: Extract Metadata (tags, labels) for Docker
212+
id: meta
213+
uses: docker/metadata-action@v5
214+
env:
215+
DOCKER_METADATA_ANNOTATIONS_LEVELS: index
216+
with:
217+
images: |
218+
${{ env.REPO_NAME }}
219+
ghcr.io/${{ env.REPO_NAME }}
220+
labels: |
221+
org.opencontainers.image.url=https://github.com/rclone/rclone/pkgs/container/rclone
222+
org.opencontainers.image.vendor=${{ github.repository_owner }}
223+
org.opencontainers.image.authors=rclone <https://github.com/rclone>
224+
org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
225+
org.opencontainers.image.revision=${{ github.sha }}
226+
tags: |
227+
type=sha
228+
type=ref,event=pr
229+
type=ref,event=branch
230+
type=semver,pattern={{version}}
231+
type=semver,pattern={{major}}
232+
type=semver,pattern={{major}}.{{minor}}
233+
type=raw,value=beta,enable={{is_default_branch}}
234+
235+
- name: Extract Tags
236+
shell: python
237+
run: |
238+
import json, os
239+
240+
metadata_json = os.environ['DOCKER_METADATA_OUTPUT_JSON']
241+
metadata = json.loads(metadata_json)
242+
243+
tags = [f"--tag '{tag}'" for tag in metadata["tags"]]
244+
tags_string = " ".join(tags)
245+
246+
with open(os.environ['GITHUB_ENV'], 'a') as env:
247+
env.write(f"TAGS={tags_string}\n")
248+
249+
- name: Extract Annotations
250+
shell: python
251+
run: |
252+
import json, os
253+
254+
metadata_json = os.environ['DOCKER_METADATA_OUTPUT_JSON']
255+
metadata = json.loads(metadata_json)
256+
257+
annotations = [f"--annotation '{annotation}'" for annotation in metadata["annotations"]]
258+
annotations_string = " ".join(annotations)
259+
260+
with open(os.environ['GITHUB_ENV'], 'a') as env:
261+
env.write(f"ANNOTATIONS={annotations_string}\n")
262+
263+
- name: Set up Docker Buildx
264+
uses: docker/setup-buildx-action@v3
265+
266+
- name: Login to Docker Hub
267+
uses: docker/login-action@v3
268+
with:
269+
username: ${{ secrets.DOCKERHUB_USERNAME }}
270+
password: ${{ secrets.DOCKERHUB_TOKEN }}
271+
272+
- name: Login to GitHub Container Registry
273+
uses: docker/login-action@v3
274+
with:
275+
registry: ghcr.io
276+
# This is the user that triggered the Workflow. In this case, it will
277+
# either be the user whom created the Release or manually triggered
278+
# the workflow_dispatch.
279+
username: ${{ github.actor }}
280+
password: ${{ secrets.GITHUB_TOKEN }}
281+
282+
- name: Create & Push Manifest List
283+
working-directory: /tmp/digests
284+
run: |
285+
docker buildx imagetools create \
286+
${{ env.TAGS }} \
287+
${{ env.ANNOTATIONS }} \
288+
$(printf 'ghcr.io/${{ env.REPO_NAME }}@sha256:%s ' *)
289+
290+
- name: Inspect and Run Multi-Platform Image
291+
run: |
292+
docker buildx imagetools inspect --raw ${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }}
293+
docker buildx imagetools inspect --raw ghcr.io/${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }}
294+
docker run --rm ghcr.io/${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }} version

0 commit comments

Comments
 (0)