Skip to content
This repository was archived by the owner on Dec 16, 2022. It is now read-only.

Commit a634c06

Browse files
hiddecohiddeco
committed
Refactor into scratch based image
The previous setup wasn't compatible with non-AMD64 builder nodes, as the published (multi-arch) image result always targeted the platform defined during build time. This commit refactors the Makefile and previous Dockerfile revision into a Makefile recipe, which can be copied over as a prerequiste step to compile and install `libgit2` with a predefined configuration before the build of a Go application. This makes the `Dockerfile` on the consumer side a bit fatter, but allows to distribute the configuration while making cross compiliation to and from multiple architectures still work. Signed-off-by: Hidde Beydals <[email protected]>
1 parent 45e789b commit a634c06

File tree

4 files changed

+136
-152
lines changed

4 files changed

+136
-152
lines changed

‎Dockerfile‎

Lines changed: 2 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,56 +1,3 @@
1-
ARG BASE_VARIANT=bullseye
2-
ARG GO_VERSION=1.16.8
3-
ARG XX_VERSION=1.0.0-rc.2
1+
FROM scratch
42

5-
FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
6-
7-
FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as gostable
8-
FROM --platform=$BUILDPLATFORM golang:1.17rc1-${BASE_VARIANT} AS golatest
9-
10-
FROM gostable AS go-linux
11-
12-
FROM go-${TARGETOS} AS build-base-bullseye
13-
14-
COPY --from=xx / /
15-
16-
RUN apt-get update && apt-get install --no-install-recommends -y clang
17-
ARG CMAKE_VERSION=3.21.3
18-
RUN curl -sL -o cmake-linux.sh "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(xx-info march).sh" \
19-
&& sh cmake-linux.sh -- --skip-license --prefix=/usr \
20-
&& rm cmake-linux.sh
21-
22-
FROM build-base-bullseye AS build-bullseye
23-
ARG TARGETPLATFORM
24-
RUN xx-apt install --no-install-recommends -y binutils gcc libc6-dev dpkg-dev
25-
26-
FROM build-${BASE_VARIANT} AS build-dependencies-bullseye
27-
28-
# Install libssh2 for $TARGETPLATFORM from "sid", as the version in "bullseye"
29-
# has been linked against gcrypt, which causes issues with PKCS* formats.
30-
# We pull (sub)dependencies from there as well, to ensure all versions are aligned,
31-
# and not accidentially linked to e.g. mbedTLS (which has limited support for
32-
# certain key formats).
33-
# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271
34-
# Ref: https://github.com/ARMmbed/mbedtls/issues/2452#issuecomment-802683144
35-
ARG TARGETPLATFORM
36-
RUN echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list \
37-
&& echo "deb-src http://deb.debian.org/debian sid main" >> /etc/apt/sources.list
38-
RUN xx-apt update \
39-
&& xx-apt -t sid install --no-install-recommends -y zlib1g-dev libssl-dev libssh2-1-dev
40-
41-
FROM build-dependencies-${BASE_VARIANT} as build-libgit2-bullseye
42-
43-
# Compile libgit2 as a dynamic build
44-
# We compile it ourselves to ensure they are properly linked with the above packages,
45-
# and to allow room for customizations (or more rapid updates than the OS).
46-
ARG LIBGIT2_PATH=/libgit2
47-
ENV LIBGIT2_PATH=${LIBGIT2_PATH}
48-
COPY hack/Makefile ${LIBGIT2_PATH}/Makefile
49-
RUN set -e; \
50-
echo "/usr/lib/$(xx-info triple)" > ${LIBGIT2_PATH}/INSTALL_LIBDIR \
51-
&& INSTALL_LIBDIR=$(cat ${LIBGIT2_PATH}/INSTALL_LIBDIR) \
52-
FLAGS=$(xx-clang --print-cmake-defines) \
53-
make -C ${LIBGIT2_PATH} \
54-
&& xx-verify $(cat ${LIBGIT2_PATH}/INSTALL_LIBDIR)/libgit2.so \
55-
&& mkdir -p ${LIBGIT2_PATH}/lib/ \
56-
&& cp -d $(cat ${LIBGIT2_PATH}/INSTALL_LIBDIR)/libgit2.so* ${LIBGIT2_PATH}/lib/
3+
COPY hack/Makefile Makefile

‎Dockerfile.test‎

Lines changed: 27 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,30 @@
1-
# This Docker image can be used to confirm the sets of dependencies can be used together with git2go.
2-
# Note: this just confirms the C library wiring is working, and no implementation details.
3-
# It does for example not tell you if ED25519 is supported by the underlying set of dependencies, etc.
4-
ARG IMG
5-
ARG TAG
6-
# NB: because the `Dockerfile` performs a build on the $BUILDPLATFORM and then later pushes it as
7-
# if it were a $TARGETPLATFORM image, we need to inverse this by NOT using --platform.
8-
FROM $IMG:$TAG as source
1+
# This Dockerfile tests the hack/Makefile output against git2go.
2+
ARG BASE_VARIANT=bullseye
3+
ARG GO_VERSION=1.16.8
4+
ARG XX_VERSION=1.0.0-rc.2
5+
6+
FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
7+
8+
FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as gostable
9+
10+
FROM gostable AS go-linux
11+
12+
FROM go-${TARGETOS} AS build-base-bullseye
13+
14+
COPY --from=xx / /
15+
COPY hack/Makefile /libgit2/Makefile
16+
17+
RUN make -C /libgit2/ cmake
18+
19+
ARG TARGETPLATFORM
20+
RUN make -C /libgit2/ dependencies
21+
22+
FROM build-base-${BASE_VARIANT} as libgit2-bullseye
23+
24+
ARG TARGETPLATFORM
25+
RUN FLAGS=$(xx-clang --print-cmake-defines) make -C /libgit2/ libgit2
26+
27+
FROM libgit2-${BASE_VARIANT} as test-bullseye
928

1029
# Cache clone
1130
ARG GIT2GO_TAG
@@ -17,17 +36,10 @@ RUN git config --global advice.detachedHead false \
1736
# Set workdir
1837
WORKDIR /git2go
1938

20-
FROM source as test
21-
2239
# Run tests
2340
ARG TARGETPLATFORM
2441
ARG CACHE_BUST
2542
RUN set -eux; \
2643
echo "=> Dynamic test at $CACHE_BUST for $TARGETPLATFORM" \
2744
&& CGO_ENABLED=1 xx-go run script/check-MakeGitError-thread-lock.go \
2845
&& CGO_ENABLED=1 xx-go test ./...
29-
30-
#RUN set -eux; \
31-
# echo "=> Static test at $CACHE_BUST for $TARGETPLATFORM" \
32-
# && CGO_ENABLED=1 xx-go run -tags "static,system_libgit2" script/check-MakeGitError-thread-lock.go \
33-
# && CGO_ENABLED=1 xx-go test -tags "static,system_libgit2" ./...

‎README.md‎

Lines changed: 40 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
# golang-with-libgit2
22

3-
This repository contains a `Dockerfile` which makes use of [`tonistiigi/xx`][xx] to produce a [Go container image][]
4-
with a dynamic set of [libgit2][] dependencies. The image can be used to build **AMD64, ARM64 and ARMv7** binaries of Go
5-
projects that depend on [git2go][].
3+
This repository contains a `Dockerfile` which `Makefile` content can be used to build the [libgit2][] dependency
4+
chain for **AMD64, ARM64 and ARMv7** binaries of Go projects that depend on [git2go][].
65

76
### :warning: **Public usage discouraged**
87

@@ -26,8 +25,8 @@ that producing a set of dependencies that work for all end-users using OS packag
2625
- [fluxcd/image-automation-controller#186](https://github.com/fluxcd/image-automation-controller/issues/186)
2726
- [fluxcd/source-controller#439](https://github.com/fluxcd/source-controller/issues/439)
2827

29-
This image is an attempt to solve (most of) these issues, by compiling `libgit2` ourselves, linking it with
30-
the required dependencies at specific versions and with specific configuration, and linker options,
28+
This image is an attempt to solve (most of) these issues, by providing the configuration to compile `libgit2` ourselves,
29+
linking it with the required dependencies at specific versions and with specific configuration, and linker options,
3130
while testing these against the git2go code before releasing the image.
3231

3332
### List of known issues
@@ -45,18 +44,9 @@ while testing these against the git2go code before releasing the image.
4544

4645
## Usage
4746

48-
To make use of the image published by the `Dockerfile`, use it as a base image for your Go build. In your application
49-
container, ensure the [runtime dependencies](#Runtime-dependencies) are present, and copy over the `libgit2` shared
50-
libraries from `$LIBGIT2_PATH/lib/*` (default `/libgit2/lib`).
51-
52-
### libgit2
53-
54-
The `libgit2` library is installed to the expected path for the `$TARGETPLATFORM`, and building a dynamically linked Go
55-
binary should be possible by just running `xx-go build`. For the application image, a copy of the `.so*` files are
56-
available in `$LIBGIT2_PATH/lib`.
57-
58-
In cases where you need to determine the architecture based library installation path without making use of `xx[-info]`,
59-
the destination path is written to `$LIBGIT2_PATH/INSTALL_LIBDIR`.
47+
To make use of the image published by the `Dockerfile`, copy over the `Makefile` from the image as a prerequisite to
48+
your Go build. Once copied over to the image, the set of targets can be used to compile `libgit2` for the
49+
`$BUILDPLATFORM` and/or `$TARGETPLATFORM` (see [example](#Dockerfile-example)).
6050

6151
### Runtime dependencies
6252

@@ -74,7 +64,25 @@ The following dependencies should be present in the image running the applicatio
7464
### `Dockerfile` example
7565

7666
```Dockerfile
77-
FROM hiddeco/golang-with-libgit2 AS build
67+
FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
68+
FROM fluxcd/golang-with-libgit2 as libgit2
69+
70+
FROM --platform=$BUILDPLATFORM golang:1.16.8-bullseye as build
71+
72+
# Copy the build utiltiies
73+
COPY --from=xx / /
74+
COPY --from=libgit2 /Makefile /libgit2/
75+
76+
# Install the libgit2 build dependencies
77+
RUN make -C /libgit2 cmake
78+
79+
ARG TARGETPLATFORM
80+
RUN make -C /libgit2 dependencies
81+
82+
# Compile and install libgit2
83+
RUN FLAGS=$(xx-clang --print-cmake-defines) make -C /libgit2 libgit2 \
84+
&& mkdir -p /libgit2/lib/ \
85+
&& cp -d /usr/lib/$(xx-info triple)/libgit2.so* /libgit2/lib/
7886

7987
# Configure workspace
8088
WORKDIR /workspace
@@ -119,13 +127,18 @@ ENTRYPOINT [ "app" ]
119127

120128
## Contributing
121129

122-
### Updating the Go version
130+
### Updating the `libgit2` version
123131

124-
In the `Dockerfile`, update the default value of the `GO_VERSION` to the new target version.
132+
Change the default value of `LIBGIT2_VERSION` in `hack/Makefile`. If applicable, change the `GIT2GO_TAG` in the
133+
`Makefile` in the repository root as well to test against another version of [git2go][].
134+
135+
### Updating the test Go version
125136

126-
### Updating the base image variant
137+
In the `Dockerfile.test`, update the default value of the `GO_VERSION` to the new target version.
127138

128-
In the `Dockerfile`, update the `BASE_VARIANT` to the new target base variant. Then, ensure all build stages making use
139+
### Updating the test base image variant
140+
141+
In the `Dockerfile.test`, update the `BASE_VARIANT` to the new target base variant. Then, ensure all build stages making use
129142
of (or depending on) the base `${BASE_VARIANT}`, use it in their `AS` stage defined for the new variant.
130143

131144
For example:
@@ -140,23 +153,16 @@ FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as go-awesom
140153
FROM go-${BASE_VARIANT} AS build-dependencies-awesome-os
141154
```
142155

143-
### Updating the `libgit2` version
144-
145-
Change the default value of `LIBGIT2_VERSION` in `hack/Makefile`. If applicable, change the `GIT2GO_TAG` in the
146-
`Makefile` in the repository root as well to test against another version of [git2go][].
147-
148156
### Releasing a new image
149157

150158
For the `main` branch, images are pushed automatically to a tag matching the branch name, and a tag in the format of
151-
`sha-<Git sha>`. In addition, images are created for new tags, with as preferred format:
152-
`<Go SemVer>-<image variant>-libgit2-<libgit2 SemVer>`.
159+
`sha-<Git sha>`. In addition, images are created for new tags, with as preferred format: `libgit2-<libgit2 SemVer>`.
153160

154-
For example, `1.16.8-bullseye-libgit2-1.1.1` for an image with
155-
**Go 1.16.8** based on **Debian bullseye** with **libgit2 1.1.1** included.
161+
For example, `libgit2-1.1.1` for an image with **libgit2 1.1.1** included.
156162

157-
In case changes happen to the `Dockerfile` while the `Go` or `libgit2` versions do not change, sequential tags should
158-
be suffixed with `-<seq num in range>`. For example, `1.16.8-bullseye-libgit2-1.1.1-2` for the **third** container image
159-
with these versions.
163+
In case changes happen to the `Dockerfile` while the `libgit2` version does not change, sequential tags should
164+
be suffixed with `-<seq num in range>`. For example, `libgit2-1.1.1-2` for the **third** container image
165+
with the same version.
160166

161167
[xx]: https://github.com/tonistiigi/xx
162168
[Go container image]: https://hub.docker.com/_/golang

‎hack/Makefile‎

Lines changed: 67 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,61 +1,99 @@
1-
BUILD_TYPE ?= "RelWithDebInfo"
2-
FLAGS ?=
1+
# This determines if we are building while making use of xx. Ensuring
2+
# the package directives target the right manager and inject (OS) vendor
3+
# specific configurations.
4+
IS_XX := $(shell command -v xx-info 2> /dev/null)
5+
ifdef IS_XX
6+
XX_VENDOR ?= $(shell xx-info vendor)
7+
endif
38

9+
# Libgit2 build configuration flags.
410
INSTALL_PREFIX ?= /usr
11+
ifndef IS_XX
512
INSTALL_LIBDIR ?= $(INSTALL_PREFIX)/lib
13+
else
14+
INSTALL_LIBDIR ?= $(INSTALL_PREFIX)/lib/$(shell xx-info triple)
15+
endif
16+
BUILD_TYPE ?= "RelWithDebInfo"
17+
FLAGS ?=
618
USE_HTTPS ?= OpenSSL
719
USE_SSH ?= ON
820

21+
# Cmake version to be installed.
22+
CMAKE_VERSION ?= 3.21.3
23+
24+
# Libgit2 version to be compiled and installed.
925
LIBGIT2_VERSION ?= 1.1.1
10-
# Overwritten to a specific version by default for git2go support
26+
# In some scenarios libgit2 needs to be checked out to a specific commit.
27+
# This takes presidence over LIBGIT_VERSION if defined.
1128
# Ref: https://github.com/libgit2/git2go/issues/834
1229
LIBGIT2_REVISION ?=
1330

14-
ifeq ($(LIBGIT2_REVISION),)
31+
# Set the download URL based on the above information.
32+
ifeq (,$(LIBGIT2_REVISION))
1533
LIBGIT2_DOWNLOAD_URL ?= https://github.com/libgit2/libgit2/archive/refs/tags/v$(LIBGIT2_VERSION).tar.gz
1634
else
1735
LIBGIT2_DOWNLOAD_URL ?= https://github.com/libgit2/libgit2/archive/$(LIBGIT2_REVISION).tar.gz
1836
endif
1937

20-
LIBGIT2_LIB := $(INSTALL_LIBDIR)/libgit2.so.$(LIBGIT2_VERSION)
21-
22-
PKG_CONFIG_PATH ?=
23-
# Detect Darwin (MacOS) to help the user a bit configuring OpenSSL,
24-
# or at least in pointing out what steps should be taken if it can't
25-
# magically figure it out by itself. Because we too, are nice people.
26-
ifeq ($(shell uname -s),Darwin)
27-
LIBGIT2_LIB := $(INSTALL_LIBDIR)/libgit2.$(LIBGIT2_VERSION).dylib
38+
# OS specific expected lib output.
39+
LIBGIT2 := $(INSTALL_LIBDIR)/libgit2.so.$(LIBGIT2_VERSION)
40+
ifeq (Darwin,$(shell uname -s))
41+
LIBGIT2 := $(INSTALL_LIBDIR)/libgit2.$(LIBGIT2_VERSION).dylib
2842
HAS_BREW := $(shell brew --version 2>/dev/null)
43+
endif
44+
45+
cmake:
46+
ifeq (debian,$(XX_VENDOR))
47+
cmake:
48+
apt-get update && apt-get install --no-install-recommends -y clang cmake
49+
endif
50+
.PHONY: cmake
51+
52+
base:
53+
ifeq (debian,$(XX_VENDOR))
54+
base:
55+
xx-apt update && xx-apt install --no-install-recommends -y binutils gcc libc6-dev dpkg-dev
56+
endif
57+
.PHONY: base
58+
59+
dependencies: base
60+
ifeq (debian,$(XX_VENDOR))
61+
# Install libssh2 for $TARGETPLATFORM from "sid", as the version in "bullseye"
62+
# has been linked against gcrypt, which causes issues with PKCS* formats.
63+
# We pull (sub)dependencies from there as well, to ensure all versions are aligned,
64+
# and not accidentially linked to e.g. mbedTLS (which has limited support for
65+
# certain key formats).
66+
# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271
67+
# Ref: https://github.com/ARMmbed/mbedtls/issues/2452#issuecomment-802683144
68+
dependencies:
69+
set -e; \
70+
echo "deb http://deb.debian.org/debian sid main" > /etc/apt/sources.list.d/sid.list \
71+
&& echo "deb-src http://deb.debian.org/debian sid main" /etc/apt/sources.list.d/sid.list \
72+
&& xx-apt update \
73+
&& xx-apt -t sid install --no-install-recommends -y zlib1g-dev libssl-dev libssh2-1-dev
74+
endif
75+
.PHONY: dependencies
76+
77+
libgit2: $(LIBGIT2)
78+
.PHONY: libgit2
79+
2980
ifdef HAS_BREW
3081
HAS_OPENSSL := $(shell brew --prefix [email protected])
3182
# NB: the OPENSSL_LDFLAGS ensures the path is included in the libgit2.pc
32-
# file. As standard brew installation doesn't appear to be system wide
33-
# on most macOS instances, and you thus have to tell explicitly where
83+
# file. As a standard brew installation doesn't appear to be system wide
84+
# on most macOS instances, and we thus have to explicitly tell where
3485
# it can be found.
3586
ifdef HAS_OPENSSL
3687
PKG_CONFIG_PATH := $(PKG_CONFIG_PATH):$(HAS_OPENSSL)/lib/pkgconfig
37-
OS_FLAGS += -DOPENSSL_LDFLAGS:STRING=-L $(HAS_OPENSSL)/lib
38-
else
39-
$(warning "Failed to detect [email protected] installation with brew. It can be installed with 'brew install [email protected]',")
40-
$(warning "or an alternative location can be provided using the PKG_CONFIG_PATH flag, for example:")
41-
$(warning "'PKG_CONFIG_PATH=/usr/local/lib/pkgconfig make'.")
88+
FLAGS += -DOPENSSL_LDFLAGS:STRING=-L $(HAS_OPENSSL)/lib
4289
endif
4390
HAS_LIBSSH2 := $(shell brew --prefix libssh2)
4491
ifdef HAS_LIBSSH2
4592
PKG_CONFIG_PATH := $(PKG_CONFIG_PATH):$(HAS_LIBSSH2)/lib/pkgconfig
46-
else
47-
$(warning "Failed to detect libssh2 installation with brew. It can be installed with 'brew install libssh2',")
48-
$(warning "or an alternative location can be provided using the PKG_CONFIG_PATH flag, for example:")
49-
$(warning "'PKG_CONFIG_PATH=/usr/local/lib/pkgconfig make'.")
50-
endif
51-
else
52-
$(warning "Failed to detect brew installation, and therefore unable to automatically determine lib paths.")
53-
$(warning "The location of openssl and libssh2 can be provided using the PKG_CONFIG_PATH flag, for example:")
54-
$(warning "'PKG_CONFIG_PATH=/usr/local/lib/pkgconfig make'.")
5593
endif
5694
endif
5795

58-
$(LIBGIT2_LIB):
96+
$(LIBGIT2):
5997
set -e; \
6098
SETUP_LIBGIT2_TMP_DIR=$$(mktemp -d) \
6199
&& curl -L $(LIBGIT2_DOWNLOAD_URL) -o $$SETUP_LIBGIT2_TMP_DIR/archive.tar.gz \
@@ -76,25 +114,6 @@ $(LIBGIT2_LIB):
76114
-DREGEX_BACKEND:STRING=builtin \
77115
-DUSE_HTTPS:STRING=$(USE_HTTPS) \
78116
-DUSE_SSH:BOOL=$(USE_SSH) \
79-
$(OS_FLAGS) \
80117
$(FLAGS) \
81118
&& cmake --build $$SETUP_LIBGIT2_TMP_DIR/build --target install \
82119
&& rm -rf $$SETUP_LIBGIT2_TMP_DIR
83-
84-
# \ && cmake -S $tmp/src -B $tmp/build $(xx-clang --print-cmake-defines) \
85-
# -DCMAKE_BUILD_TYPE:STRING=$LIBGIT2_BUILD_TYPE \
86-
# -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON \
87-
# -DCMAKE_C_FLAGS=-fPIC \
88-
# -DCMAKE_CXX_FLAGS=-fPIC \
89-
# -DDEPRECATE_HARD=ON \
90-
# -DCMAKE_INSTALL_PREFIX:PATH=/usr \
91-
# -DCMAKE_INSTALL_LIBDIR:PATH=/usr/lib/$(xx-info triple) \
92-
# -DBUILD_CLAR:BOOL:BOOL=OFF \
93-
# -DTHREADSAFE:BOOL=ON \
94-
# -DBUILD_SHARED_LIBS=OFF \
95-
# -DUSE_BUNDLED_ZLIB:BOOL=OFF \
96-
# -DUSE_HTTP_PARSER:STRING=builtin \
97-
# -DREGEX_BACKEND:STRING=builtin \
98-
# -DUSE_HTTPS:STRING=OpenSSL \
99-
# -DUSE_SSH:BOOL=ON \
100-
# && cmake --build $tmp/build --target install \

0 commit comments

Comments
 (0)