Merge pull request #202 from smallstep/jdoss/Packages

Enable RPM and Deb package uploads to packages.smallstep.com

Author: jdoss

.github/workflows/release.yml

@@ -36,6 +36,8 @@ jobs:
           OUT=$?
           if [ $OUT -eq 0 ]; then IS_PRERELEASE=true; else IS_PRERELEASE=false; fi
           echo "IS_PRERELEASE=${IS_PRERELEASE}" >> "${GITHUB_OUTPUT}"
+          echo "IS_PRERELEASE=${IS_PRERELEASE}" >> "${GITHUB_ENV}"
+
       - name: Extract Tag Names
         id: extract-tag
         run: |
@@ -46,6 +48,7 @@ jobs:
           echo "DOCKER_TAGS_CLOUD=${{ env.DOCKER_IMAGE }}:${VERSION}-${CLOUD_TAG}" >> "${GITHUB_ENV}"
           echo "DOCKER_TAGS_DEBIAN=${{ env.DOCKER_IMAGE }}:${VERSION}-${DEBIAN_TAG}" >> "${GITHUB_ENV}"
           echo "DOCKER_TAGS_WOLFI=${{ env.DOCKER_IMAGE }}:${VERSION}-${WOLFI_TAG}" >> "${GITHUB_ENV}"
+
       - name: Add Latest Tag
         if: steps.is_prerelease.outputs.IS_PRERELEASE == 'false'
         run: |
@@ -54,6 +57,7 @@ jobs:
           echo "DOCKER_TAGS_CLOUD=${{ env.DOCKER_IMAGE }}:${CLOUD_TAG}" >> "${GITHUB_ENV}"
           echo "DOCKER_TAGS_DEBIAN=${{ env.DOCKER_IMAGE }}:${DEBIAN_TAG}" >> "${GITHUB_ENV}"
           echo "DOCKER_TAGS_WOLFI=${{ env.DOCKER_IMAGE }}:${WOLFI_TAG}" >> "${GITHUB_ENV}"
+
       - name: Create Release
         id: create_release
         uses: softprops/action-gh-release@c95fe1489396fe8a9eb87c0abf8aa5b2ef267fda # v2.2.1
@@ -69,17 +73,86 @@ jobs:
     name: Upload Assets to Github w/ goreleaser
     runs-on: ubuntu-latest
     needs: create_release
+    permissions:
+      id-token: write
+      contents: write
+      packages: write
+    env:
+      GPG_PRIVATE_KEY_FILE: "0x889B19391F774443-Certify.key"
     steps:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 0
+
       - name: setup release environment
         run: |-
+          # shellcheck disable=SC2129
           echo 'GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}' > .release-env
-      - name: release publish
+          { echo 'GORELEASER_KEY=${{ secrets.GORELEASER_KEY }}'; \
+          echo 'AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}'; \
+          echo 'AWS_S3_BUCKET=${{ secrets.AWS_S3_BUCKET }}'; \
+          echo 'AWS_S3_REGION=${{ secrets.AWS_S3_REGION }}'; \
+          echo 'AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}'; \
+          echo 'GPG_PRIVATE_KEY_FILE=${{ env.GPG_PRIVATE_KEY_FILE }}'; \
+          echo 'NFPM_PASSPHRASE=${{ secrets.GPG_PRIVATE_KEY_PASSWORD }}'; } >> .release-env
+
+      - name: Write GPG private key to file
+        run: |
+          echo "${GPG_PRIVATE_KEY}" > "${GPG_PRIVATE_KEY_FILE}"
+        shell: bash
+        env:
+          GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }}
+
+      - name: Build binaries
         run: make release
 
+      - name: Authenticate to Google Cloud
+        if: ${{ needs.create_release.outputs.is_prerelease == 'false' }}
+        id: gcloud-auth
+        uses: google-github-actions/auth@v2
+        with:
+          token_format: access_token
+          workload_identity_provider: ${{ secrets.GOOGLE_CLOUD_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ secrets.GOOGLE_CLOUD_GITHUB_SERVICE_ACCOUNT }}
+
+      - name: Set up Google Cloud SDK
+        if: ${{ needs.create_release.outputs.is_prerelease == 'false' }}
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          project_id: ${{ secrets.GOOGLE_CLOUD_PACKAGES_PROJECT_ID }}
+
+      - name: Get Release Date
+        id: release_date
+        run: |
+          # shellcheck disable=SC2129
+          RELEASE_DATE=$(date -u +"%y-%m-%d")
+          echo "RELEASE_DATE=${RELEASE_DATE}" >> "${GITHUB_ENV}"
+          echo 'IS_PRERELEASE=${{ needs.create_release.outputs.is_prerelease }}' >> "${GITHUB_ENV}"
+
+      - name: Run GoReleaser Pro
+        uses: goreleaser/[email protected]
+        with:
+          distribution: goreleaser-pro
+          version: v2.8.1
+          args: publish
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+          AWS_S3_REGION: ${{ secrets.AWS_S3_REGION }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GORELEASER_PAT }}
+          GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
+          NFPM_PASSPHRASE: ${{ secrets.GPG_PRIVATE_KEY_PASSWORD }}
+          RELEASE_DATE: ${{ env.RELEASE_DATE }}
+          IS_PRERELEASE: ${{ needs.create_release.outputs.is_prerelease }}
+
+      - name: Shred and remove GPG private key
+        run: |
+          shred -zun 3 "${GPG_PRIVATE_KEY_FILE}"
+          shred -zun 3 .release-env
+        shell: bash
+
   build_upload_docker:
     name: Build & Upload Docker Image
     needs: create_release

.gitignore

@@ -25,3 +25,7 @@ dist/
 
 # Dependency directories (remove the comment below to include it)
 # vendor/
+
+# Packages files
+0x889B19391F774443-Certify.key
+gha-creds-*.json

.goreleaser.yml

@@ -4,6 +4,19 @@
 #  - https://github.com/goreleaser/goreleaser-cross
 #  - https://github.com/goreleaser/goreleaser-cross-example
 project_name: step-kms-plugin
+version: 2
+
+variables:
+  packageName: step-kms-plugin
+  packageRelease: 1 # Manually update release: in the nfpm section to match this value if you change this
+
+after:
+  hooks:
+    # This script depends on IS_PRERELEASE env being set. This is set by CI in the Is Pre-release step.
+    - cmd: bash scripts/package-repo-import.sh {{ .Var.packageName }} {{ .Version }}
+      output: true
+      env:
+      - IS_PRERELEASE={{ .Env.IS_PRERELEASE }}
 
 builds:
   - id: linux-amd64
@@ -113,11 +126,17 @@ archives:
       - completions/*
 
 nfpms:
-  - builds:
+  - id: packages
+    builds:
       - linux-amd64
       - linux-arm64
-    package_name: step-kms-plugin
-    file_name_template: "{{ .PackageName }}_{{ .Version }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}{{ if .Mips }}_{{ .Mips }}{{ end }}"
+    package_name: "{{ .Var.packageName }}"
+    release: "1"
+    file_name_template: >-
+      {{- trimsuffix .ConventionalFileName .ConventionalExtension -}}
+      {{- if and (eq .Arm "6") (eq .ConventionalExtension ".deb") }}6{{ end -}}
+      {{- if not (eq .Amd64 "v1")}}{{ .Amd64 }}{{ end -}}
+      {{- .ConventionalExtension -}}
     vendor: Smallstep Labs
     homepage: https://github.com/smallstep/step-kms-plugin
     maintainer: Smallstep <[email protected]>
@@ -139,6 +158,13 @@ nfpms:
       - src: completions/zsh_completion
         dst: /usr/share/zsh/site-functions/_step-kms-plugin
         packager: rpm
+    rpm:
+      signature:
+          key_file: "{{ .Env.GPG_PRIVATE_KEY_FILE }}"
+    deb:
+      signature:
+          key_file: "{{ .Env.GPG_PRIVATE_KEY_FILE }}"
+          type: origin
     overrides:
       deb:
         dependencies:
@@ -160,6 +186,14 @@ sboms:
 checksum:
   name_template: "checksums.txt"
 
+publishers:
+- name: Google Cloud Artifact Registry
+  ids:
+  - packages
+  cmd: ./scripts/package-upload.sh {{ abs .ArtifactPath }} {{ .Var.packageName }} {{ .Version }} {{ .Var.packageRelease }}
+  env:
+  - IS_PRERELEASE={{ .Env.IS_PRERELEASE }}
+
 snapshot:
   version_template: "{{ .Tag }}"
 

Makefile

@@ -10,6 +10,9 @@ GOOS_OVERRIDE ?=
 OUTPUT_ROOT=output/
 RELEASE=./.releases
 
+DOCKER_HOST ?= /var/run/docker.sock
+DOCKER_SOCK := $(if $(filter unix://%,$(DOCKER_HOST)),$(patsubst unix://%,%,$(DOCKER_HOST)),$(DOCKER_HOST))
+
 #########################################
 # Default
 #########################################
@@ -115,20 +118,25 @@ govulncheck:
 
 release-dev:
 	$Q @docker run -it --rm --privileged -e CGO_ENABLED=1 \
+		-e GORELEASER_KEY=$(GORELEASER_KEY) \
+		-e IS_PRERELEASE=true \
 		--entrypoint /bin/bash \
-		-v /var/run/docker.sock:/var/run/docker.sock \
-		-v `pwd`:/go/src/$(PKG) \
+		-v $(DOCKER_SOCK):/var/run/docker.sock:Z \
+		-v `pwd`:/go/src/$(PKG):Z \
 		-w /go/src/$(PKG) \
-		ghcr.io/goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION}
+		ghcr.io/goreleaser/goreleaser-cross-pro:${GOLANG_CROSS_VERSION}
 
 release-dry-run:
 	$Q @docker run --rm --privileged -e CGO_ENABLED=1 \
+		-e GORELEASER_KEY=$(GORELEASER_KEY) \
+		-e GPG_PRIVATE_KEY_FILE=/dev/null \
+		-e IS_PRERELEASE=true \
 		--entrypoint /go/src/$(PKG)/docker/build/entrypoint.sh \
-		-v /var/run/docker.sock:/var/run/docker.sock \
-		-v `pwd`:/go/src/$(PKG) \
+		-v $(DOCKER_SOCK):/var/run/docker.sock:Z \
+		-v `pwd`:/go/src/$(PKG):Z \
 		-w /go/src/$(PKG) \
-		ghcr.io/goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \
-		--clean --skip=validate --skip=publish
+		ghcr.io/goreleaser/goreleaser-cross-pro:${GOLANG_CROSS_VERSION} \
+		release --clean --skip=validate --skip=sign --prepare
 
 release:
 	@if [ ! -f ".release-env" ]; then \
@@ -137,11 +145,12 @@ release:
 	fi
 	$Q @docker run --rm --privileged -e CGO_ENABLED=1 --env-file .release-env \
 		--entrypoint /go/src/$(PKG)/docker/build/entrypoint.sh \
-		-v /var/run/docker.sock:/var/run/docker.sock \
+		-v ./$(GPG_PRIVATE_KEY_FILE):/$(GPG_PRIVATE_KEY_FILE) \
+		-v $(DOCKER_SOCK):/var/run/docker.sock \
 		-v `pwd`:/go/src/$(PKG) \
 		-w /go/src/$(PKG) \
-		ghcr.io/goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \
-		release --clean
+		ghcr.io/goreleaser/goreleaser-cross-pro:${GOLANG_CROSS_VERSION} \
+		release --clean --prepare
 
 .PHONY: release-dev release-dry-run release
 

scripts/package-repo-import.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+set -e
+
+: ${GCLOUD_LOCATION:=us-central1}
+: ${GCLOUD_RPM_REPO:=rpms}
+: ${GCLOUD_DEB_REPO:=debs}
+
+PACKAGE="${1}"
+VERSION="${2}"
+RELEASE="1"
+EPOCH="0"
+GORELEASER_PHASE=${GORELEASER_PHASE:-release}
+
+echo "Package: ${PACKAGE}"
+echo "Version: ${VERSION}"
+
+check_package() {
+  local EXITCODE=0
+  local REPO="${1}"
+  local VER="${2}"
+  if [ ! -f /tmp/version-deleted.stamp ]; then
+    gcloud artifacts versions list \
+       --repository "${REPO}" \
+       --location "${GCLOUD_LOCATION}" \
+       --package "${PACKAGE}" \
+       --filter "VERSION:${VER}" \
+       --format json  2> /dev/null \
+       | jq -re '.[].name?' >/dev/null 2>&1 \
+       || EXITCODE=$?
+    if [[ "${EXITCODE}" -eq 0 ]]; then
+      echo "Package version already exists. Removing it..."
+      gcloud artifacts versions delete \
+      --quiet "${VER}" \
+      --package "${PACKAGE}" \
+      --repository "${REPO}" \
+      --location "${GCLOUD_LOCATION}"
+      touch /tmp/version-deleted.stamp
+    fi
+  fi
+}
+
+if [[ ${IS_PRERELEASE} == "true" ]]; then
+  echo "Skipping artifact import; IS_PRERELEASE is 'true'"
+  exit 0;
+fi
+
+check_package "${GCLOUD_RPM_REPO}" "${EPOCH}:${VERSION}-${RELEASE}"
+gcloud artifacts yum import "${GCLOUD_RPM_REPO}" \
+  --location "${GCLOUD_LOCATION}" \
+  --gcs-source "gs://artifacts-outgoing/${PACKAGE}/rpm/${VERSION}/*"
+
+check_package ${GCLOUD_DEB_REPO} "${VERSION}-${RELEASE}"}
+gcloud artifacts apt import "${GCLOUD_DEB_REPO}" \
+  --location "${GCLOUD_LOCATION}" \
+  --gcs-source "gs://artifacts-outgoing/${PACKAGE}/deb/${VERSION}/*"

scripts/package-upload.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+FILE="${1}"
+PACKAGE="${2}"
+VERSION="${3}"
+
+
+echo "Package File: ${FILE}"
+echo "Package: ${PACKAGE}"
+echo "Version: ${VERSION}"
+echo "Prerelease: ${IS_PRERELEASE}"
+
+if [[ ${IS_PRERELEASE} == "true" ]]; then
+  echo "Skipping artifact upload; IS_PRERELEASE is 'true'"
+  exit 0;
+fi
+
+if [ "${FILE: -4}" == ".deb" ]; then
+  if [[ "${FILE}" =~ "armhf6" ]]; then
+    echo "Skipping ${FILE} due to GCP Artifact Registry armhf conflict!"
+  else
+    gcloud storage cp ${FILE} gs://artifacts-outgoing/${PACKAGE}/deb/${VERSION}/
+  fi
+else
+  gcloud storage cp ${FILE} gs://artifacts-outgoing/${PACKAGE}/rpm/${VERSION}/
+fi