Build ffi on alpine (#65)

* Build ffi on alpine

* Add arm build - improve caching

Author: pawelchcki

.dockerignore

@@ -0,0 +1,6 @@
+target/
+build/
+.github/
+Dockerfile.build
+.git/
+docker-bake.hcl

.github/buildkitd.toml

@@ -0,0 +1,3 @@
+[worker.oci]
+# Set max parallelism for buildx bake workers to prevent builds running slower due to too many steps being executed at the same time
+  max-parallelism = 4

.github/workflows/test.yml

@@ -94,3 +94,32 @@ jobs:
           cd examples/ffi/build
           cmake -S .. -DDatadog_ROOT=$OUTPUT_FOLDER
           cmake --build .
+
+  ffi_bake:
+    strategy:
+      matrix:
+        target: [alpine-build] # debian-build-aarch64 is oom killed at the moment
+    name: "FFI ${{ matrix.target }} via docker bake"
+
+    concurrency: 
+      group: ci-${{ github.ref }}-${{ matrix.target }}
+      cancel-in-progress: true
+
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          config: .github/buildkitd.toml
+      -
+        name: Build and Generate FFI
+        uses: docker/bake-action@v2
+        with:
+          targets: ${{ matrix.target }}
+          set: |
+            *.cache-from=type=gha,scope=${{ matrix.target }}
+            *.cache-to=type=gha,mode=max,scope=${{ matrix.target }}

docker-bake.hcl

@@ -0,0 +1,37 @@
+target "alpine-base" {
+  dockerfile = "tools/docker/Dockerfile.build"
+  tags = ["ghcr.io/datadog/libdatadog-build:alpine-base"]
+  target = "alpine_builder"
+}
+
+target "alpine-build" {
+  dockerfile = "tools/docker/Dockerfile.build"
+  args = {
+    BUILDER_IMAGE = "alpine_builder"
+  }
+  target = "ffi_build_output"
+  platforms = ["linux/amd64"]
+  output = ["build/x86_64-alpine-linux-musl"]
+}
+
+target "debian-build" {
+  dockerfile = "tools/docker/Dockerfile.build"
+  args = {
+    BUILDER_IMAGE = "debian_builder"
+  }
+  target = "ffi_build_output"
+  platforms = ["linux/amd64"]
+  output = ["build/x86_64-unknown-linux-gnu"]
+}
+
+target "alpine-build-aarch64" {
+  inherits = ["alpine-build"]
+  platforms = ["linux/arm64"]
+  output = ["build/aarch64-alpine-linux-musl"]
+}
+
+target "debian-build-aarch64" {
+  inherits = ["debian-build"]
+  platforms = ["linux/arm64"]
+  output = ["build/aarch64-unknown-linux-gnu"]
+}

tools/docker/Dockerfile.build

@@ -0,0 +1,106 @@
+ARG ALPINE_BASE_IMAGE="alpine:3.16"
+ARG CARGO_BUILD_INCREMENTAL="true"
+ARG CARGO_NET_RETRY="2"
+ARG BUILDER_IMAGE=debian_builder
+
+### Debian builder
+FROM rust:1-slim-buster as debian_builder
+ENV CARGO_HOME="/root/.cargo"
+WORKDIR /build
+RUN cargo install cbindgen; mv /root/.cargo/bin/cbindgen /usr/bin/; rm -rf /root/.cargo
+
+### Debian buildplatform builder
+FROM --platform=$BUILDPLATFORM rust:1-slim-buster as debian_builder_platform_native
+ENV CARGO_HOME="/root/.cargo"
+WORKDIR /build
+
+### Alpine builder
+FROM ${ALPINE_BASE_IMAGE} as alpine_base
+ENV CARGO_HOME="/root/.cargo"
+WORKDIR /build
+
+RUN apk update \
+  && apk add --no-cache \
+    build-base \
+    cargo \
+    curl \
+    git \
+    make \
+    patchelf \
+    protoc \
+    pkgconf \
+    unzip \
+    bash \
+  && mkdir /usr/local/src
+
+# Tell docker to use bash as the default
+SHELL ["/bin/bash", "-c"]
+
+# Don't use rustup! For some reason it provides different native-static-libs
+# and this can cause problems for users.
+# Also, it doesn't understand x86_64-alpine-linux-musl like the OS's cargo.
+#RUN rustup-init -y --no-modify-path --default-toolchain stable
+
+FROM alpine_base as alpine_aws_cli
+RUN apk add --no-cache \
+    python3 \
+    py3-pip \
+    groff \
+  && pip3 install --upgrade pip \
+  && pip3 install --no-cache-dir \
+    awscli \
+  && rm -rf /var/cache/apk/*
+
+RUN aws --version   # Just to make sure its installed alright
+
+FROM alpine_base as alpine_cbindgen
+ENV PATH="/root/.cargo/bin:$PATH"
+ARG CARGO_BUILD_INCREMENTAL
+ARG CARGO_NET_RETRY
+ENV CARGO_NET_RETRY="${CARGO_NET_RETRY}"
+RUN cargo install cbindgen && rm -rf /root/.cargo/registry /root/.cargo/git
+
+FROM alpine_aws_cli as alpine_builder
+COPY --from=alpine_cbindgen /root/.cargo/bin/cbindgen /usr/local/bin/cbindgen
+
+
+### Cache cargo metadata between builds
+FROM debian_builder_platform_native AS ffi_build_platform_agnostic_cache_build
+# update cache cargo.io metadata
+RUN cargo search nothing --limit 1 
+
+# create stubs to cache compilation of dependendencies
+COPY [ "Cargo.lock", "Cargo.toml", "./"]
+COPY "ddcommon/Cargo.toml" "ddcommon/" 
+COPY "ddcommon-ffi/Cargo.toml" "ddcommon-ffi/" 
+COPY "ddtelemetry/Cargo.toml" "ddtelemetry/"
+COPY "ddtelemetry-ffi/Cargo.toml" "ddtelemetry-ffi/"
+COPY "profiling/Cargo.toml" "profiling/"
+COPY "profiling-ffi/Cargo.toml" "profiling-ffi/"
+COPY "tools/Cargo.toml" "tools/" 
+RUN find -name "Cargo.toml" | sed -e s#Cargo.toml#src/lib.rs#g | xargs -n 1 sh -c 'mkdir -p $(dirname $1); touch $1; echo $1' create_stubs
+RUN echo tools/src/bin/dedup_headers.rs ddtelemetry/examples/tm-worker-test.rs | xargs -n 1 sh -c 'mkdir -p $(dirname $1); touch $1; echo $1' create_stubs
+
+# cache dependencies
+RUN cargo fetch --locked
+
+# extract cargo cache
+FROM --platform=$BUILDPLATFORM scratch as ffi_build_platform_agnostic_cache
+COPY --from=ffi_build_platform_agnostic_cache_build /root/.cargo /root/.cargo
+COPY --from=ffi_build_platform_agnostic_cache_build /build /build
+
+### FFI builder
+FROM ${BUILDER_IMAGE} AS ffi_build
+COPY --from=ffi_build_platform_agnostic_cache /root/.cargo /root/.cargo/
+COPY --from=ffi_build_platform_agnostic_cache /build /build
+WORKDIR /build
+# cache debug dependency build
+RUN cargo build --lib --all
+# cache release dependency build
+RUN cargo build --release --lib --all
+
+COPY ./ ./
+RUN ./build-profiling-ffi.sh /build/output
+
+FROM scratch as ffi_build_output
+COPY --from=ffi_build /build/output/ ./