Merge pull request #661 from lightninglabs/add-github-cd-release-build

Add GitHub CD release build workflow

Author: Roasbeef

.github/workflows/main.yaml

@@ -19,12 +19,6 @@ env:
   # go needs absolute directories, using the $HOME variable doesn't work here.
   GOPATH: /home/runner/work/go
 
-  # If you change this value, please change it in the following files as well:
-  # /Dockerfile
-  # /dev.Dockerfile
-  # /make/builder.Dockerfile
-  # /taprpc/Dockerfile
-  # /tools/Dockerfile
   GO_VERSION: '1.21.0'
 
 jobs:

.github/workflows/release.yaml

@@ -0,0 +1,142 @@
+name: Release build
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  GO_VERSION: 1.21.0
+
+jobs:
+  main:
+    name: Release build
+    runs-on: ubuntu-latest
+    steps:
+      - name: git checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: setup go ${{ env.GO_VERSION }}
+        uses: actions/setup-go@v2
+        with:
+          go-version: '${{ env.GO_VERSION }}'
+
+      - name: Set env
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+
+      - name: build release for all architectures
+        run: SKIP_VERSION_CHECK=1 make release tag=${{ env.RELEASE_VERSION }}
+
+      - name: Create Release
+        uses: lightninglabs/gh-actions/[email protected]
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ env.RELEASE_VERSION }}
+          name: tapd ${{ env.RELEASE_VERSION }}
+          draft: true
+          prerelease: false
+          files: tapd-${{ env.RELEASE_VERSION }}/*
+          body: |
+            # Database Migrations
+            TODO
+
+            # Verifying the Release
+
+            In order to verify the release, you'll need to have `gpg` or `gpg2` installed on your system. Once you've obtained a copy (and hopefully verified that as well), you'll first need to import the keys that have signed this release if you haven't done so already: 
+
+            ```
+            curl https://raw.githubusercontent.com/lightninglabs/taproot-assets/main/scripts/keys/roasbeef.asc | gpg --import
+            ```
+
+            Once you have the required PGP keys, you can verify the release (assuming `manifest-roasbeef-${{ env.RELEASE_VERSION }}.sig` and `manifest-${{ env.RELEASE_VERSION }}.txt` are in the current directory) with:
+
+            ```
+            gpg --verify manifest-roasbeef-${{ env.RELEASE_VERSION }}.sig manifest-${{ env.RELEASE_VERSION }}.txt
+            ```
+
+            You should see the following if the verification was successful:
+
+            ```
+            gpg: Signature made Wed Sep 30 17:35:20 2020 PDT
+            gpg:                using RSA key 60A1FA7DA5BFF08BDCBBE7903BBD59E99B280306
+            gpg: Good signature from "Olaoluwa Osuntokun <[email protected]>" [ultimate]
+            ```
+
+            That will verify the signature of the manifest file, which ensures integrity and authenticity of the archive you've downloaded locally containing the binaries. Next, depending on your operating system, you should then re-compute the `sha256` hash of the archive with `shasum -a 256 <filename>`, compare it with the corresponding one in the manifest file, and ensure they match *exactly*.
+
+            ## Verifying the Release Timestamp 
+
+            From this new version onwards, in addition time-stamping the _git tag_ with [OpenTimestamps](https://opentimestamps.org/), we'll also now timestamp the manifest file along with its signature. Two new files are now included along with the rest of our release artifacts: ` manifest-roasbeef-${{ env.RELEASE_VERSION }}.txt.asc.ots`.
+
+            Assuming you have the opentimestamps client installed locally, the timestamps can be verified with the following commands: 
+            ```
+            ots verify manifest-roasbeef-${{ env.RELEASE_VERSION }}.sig.ots -f manifest-roasbeef-${{ env.RELEASE_VERSION }}.sig
+            ```
+
+            Alternatively, [the OpenTimestamps website](https://opentimestamps.org/) can be used to verify timestamps if one doesn't have a `bitcoind` instance accessible locally.
+
+            These timestamps should give users confidence in the integrity of this release even after the key that signed the release expires. 
+
+            ## Verifying the Release Binaries
+
+            Our release binaries are fully reproducible. Third parties are able to verify that the release binaries were produced properly without having to trust the release manager(s). See our [reproducible builds guide](https://github.com/lightninglabs/taproot-assets/blob/main/docs/release.md) for how this can be achieved.
+            The release binaries are compiled with `go${{ env.GO_VERSION }}`, which is required by verifiers to arrive at the same ones.
+
+            The `make release` command can be used to ensure one rebuilds with all the same flags used for the release. If one wishes to build for only a single platform, then `make release sys=<OS-ARCH> tag=<tag>` can be used. 
+
+            Finally, you can also verify the _tag_ itself with the following command:
+
+            ```
+            $ git verify-tag ${{ env.RELEASE_VERSION }}
+            gpg: Signature made Tue Sep 15 18:55:00 2020 PDT
+            gpg:                using RSA key 60A1FA7DA5BFF08BDCBBE7903BBD59E99B280306
+            gpg: Good signature from "Olaoluwa Osuntokun <[email protected]>" [ultimate]
+            ```
+
+            ## Verifying the Docker Images
+
+            To verify the `tapd` and `tapcli` binaries inside the docker images against the signed, reproducible release binaries, there is a verification script in the image that can be called (before starting the container for example):
+
+            ```shell
+            $ docker run --rm --entrypoint="" lightninglabs/taproot-assets:${{ env.RELEASE_VERSION }} /verify-install.sh ${{ env.RELEASE_VERSION }}
+            $ OK=$?
+            $ if [ "$OK" -ne "0" ]; then echo "Verification failed!"; exit 1; done
+            $ docker run lightninglabs/taproot-assets [command-line options]
+            ```
+
+            # Building the Contained Release 
+
+            Users are able to rebuild the target release themselves without having to fetch any of the dependencies. In order to do so, assuming
+            that `vendor.tar.gz` and `tapd-source-${{ env.RELEASE_VERSION }}.tar.gz` are in the current directory, follow these steps:
+
+            ```
+            tar -xvzf vendor.tar.gz
+            tar -xvzf tapd-source-${{ env.RELEASE_VERSION }}.tar.gz
+            GO111MODULE=on go install -v -mod=vendor -ldflags "-X github.com/lightninglabs/taproot-assets/build.Commit=${{ env.RELEASE_VERSION }}" ./cmd/tapd
+            GO111MODULE=on go install -v -mod=vendor -ldflags "-X github.com/lightninglabs/taproot-assets/build.Commit=${{ env.RELEASE_VERSION }}" ./cmd/tapcli
+            ```
+
+            The `-mod=vendor` flag tells the `go build` command that it doesn't need to fetch the dependencies, and instead, they're all enclosed in the local vendor directory. 
+
+            Additionally, it's now possible to use the enclosed `release.sh` script to bundle a release for a _specific_ system like so: 
+
+            ```
+            make release sys="linux-arm64 darwin-amd64"
+            ```
+
+            ⚡️⚡️⚡️ OK, now to the rest of the release notes! ⚡️⚡️⚡️
+
+            # Release Notes
+
+            TODO
+
+            # Contributors (Alphabetical Order)
+
+            TODO

Dockerfile

@@ -1,10 +1,3 @@
-# If you change this value, please change it in the following files as well:
-# /.github/workflows/main.yaml
-# /dev.Dockerfile
-# /make/builder.Dockerfile
-# /taprpc/Dockerfile
-# /tools/Dockerfile
-# /itest/loadtest/Dockerfile
 FROM golang:1.21.0-alpine as builder
 
 # Force Go to use the cgo based DNS resolver. This is required to ensure DNS

Makefile

@@ -57,6 +57,8 @@ DOCKER_TOOLS = docker run \
   -v $(shell bash -c "go env GOMODCACHE || (mkdir -p /tmp/go-modcache; echo /tmp/go-modcache)"):/tmp/build/.modcache \
   -v $$(pwd):/build taproot-assets-tools
 
+GO_VERSION = 1.21.0
+
 GREEN := "\\033[0;32m"
 NC := "\\033[0m"
 define print
@@ -251,10 +253,20 @@ fmt: $(GOIMPORTS_BIN)
 	@$(call print, "Formatting source.")
 	gofmt -l -w -s $(GOFILES_NOVENDOR)
 
-lint: docker-tools
+check-go-version-yaml:
+	@$(call print, "Checking for target Go version (v$(GO_VERSION)) in  YAML files (*.yaml, *.yml)")
+	./tools/check-go-version-yaml.sh $(GO_VERSION)
+
+check-go-version-dockerfile:
+	@$(call print, "Checking for target Go version (v$(GO_VERSION)) in Dockerfile files (*Dockerfile)")
+	./tools/check-go-version-dockerfile.sh $(GO_VERSION)
+
+lint-source: docker-tools
 	@$(call print, "Linting source.")
 	$(DOCKER_TOOLS) golangci-lint run -v $(LINT_WORKERS)
 
+lint: lint-source check-go-version-dockerfile check-go-version-yaml
+
 list:
 	@$(call print, "Listing commands.")
 	@$(MAKE) -qp | \

dev.Dockerfile

@@ -1,10 +1,4 @@
-# If you change this value, please change it in the following files as well:
-# /.github/workflows/main.yaml
-# /Dockerfile
-# /make/builder.Dockerfile
-# /taprpc/Dockerfile
-# /tools/Dockerfile
-FROM golang:1.21.0 as builder 
+FROM golang:1.21.0 as builder
 
 WORKDIR /app
 

docs/release.md

@@ -0,0 +1,125 @@
+# `taproot-assets`'s Reproducible Build System
+
+This package contains the build script that the `taproot-assets` project uses in
+order to build binaries for each new release. As of `go1.13`, with some new
+build flags, binaries are now reproducible, allowing developers to build the
+binary on distinct machines, and end up with a byte-for-byte identical binary.
+However, this wasn't _fully_ solved in `go1.13`, as the build system still
+includes the directory the binary is built into the binary itself. As a result,
+our scripts utilize a work around needed until `go1.13.2`.  
+
+## Building a New Release
+
+### MacOS
+
+The first requirement is to have [`docker`](https://www.docker.com/)
+installed locally and running. The second requirement is to have `make`
+installed. Everything else (including `golang`) is included in the release
+helper image.
+
+To build a release, run the following commands:
+
+```shell
+$  git clone https://github.com/lightninglabs/taproot-assets.git
+$  cd taproot-assets
+$  git checkout <TAG> # <TAG> is the name of the next release/tag
+$  make docker-release tag=<TAG>
+```
+
+Where `<TAG>` is the name of the next release of `taproot-assets`.
+
+### Linux/Windows (WSL)
+
+No prior set up is needed on Linux or macOS is required in order to build the
+release binaries. However, on Windows, the only way to build the release
+binaries at the moment is by using the Windows Subsystem Linux. One can build
+the release binaries following these steps:
+
+```shell
+$  git clone https://github.com/lightninglabs/taproot-assets.git
+$  cd taproot-assets
+$  git checkout <TAG> # <TAG> is the name of the next release/tag
+$  make release tag=<TAG>
+```
+
+This will then create a directory of the form `taproot-assets-<TAG>` containing
+archives of the release binaries for each supported operating system and
+architecture, and a manifest file containing the hash of each archive.
+
+## Verifying a Release
+
+With `go1.13`, it's now possible for third parties to verify release binaries.
+Before this version of `go`, one had to trust the release manager(s) to build the
+proper binary. With this new system, third parties can now _independently_ run
+the release process, and verify that all the hashes of the release binaries
+match exactly that of the release binaries produced by said third parties.
+
+To verify a release, one must obtain the following tools (many of these come
+installed by default in most Unix systems): `gpg`/`gpg2`, `shashum`, and
+`tar`/`unzip`.
+
+Once done, verifiers can proceed with the following steps:
+
+1. Acquire the archive containing the release binaries for one's specific
+   operating system and architecture, and the manifest file along with its
+   signature.
+2. Verify the signature of the manifest file with `gpg --verify
+   manifest-<TAG>.txt.sig`. This will require obtaining the PGP keys which
+   signed the manifest file, which are included in the release notes.
+3. Recompute the `SHA256` hash of the archive with `shasum -a 256 <filename>`,
+   locate the corresponding one in the manifest file, and ensure they match
+   __exactly__.
+
+At this point, verifiers can use the release binaries acquired if they trust
+the integrity of the release manager(s). Otherwise, one can proceed with the
+guide to verify the release binaries were built properly by obtaining `shasum`
+and `go` (matching the same version used in the release):
+
+4. Extract the release binaries contained within the archive, compute their
+   hashes as done above, and note them down.
+5. Ensure `go` is installed, matching the same version as noted in the release
+   notes. 
+6. Obtain a copy of `taproot-assets`'s source code with `git clone
+   https://github.com/lightninglabs/taproot-assets` and checkout the source code of the
+   release with `git checkout <TAG>`.
+7. Proceed to verify the tag with `git verify-tag <TAG>` and compile the
+   binaries from source for the intended operating system and architecture with
+   `make release sys=OS-ARCH tag=<TAG>`.
+8. Extract the archive found in the `taproot-assets-<TAG>` directory created by
+   the release script and recompute the `SHA256` hash of the release binaries
+   (`tapd` and `tapcli`) with `shasum -a 256 <filename>`. These should match
+   __exactly__ as the ones noted above.
+
+## Verifying Docker Images
+
+To verify the `tapd` and `tapcli` binaries inside the
+[official provided docker images](https://hub.docker.com/r/lightninglabs/taproot-assets)
+against the signed, reproducible release binaries, there is a verification
+script in the image that can be called (before starting the container for
+example):
+
+```shell
+$  docker run --rm --entrypoint="" lightninglabs/taproot-assets:v0.3.0 /verify-install.sh v0.3.0
+$  OK=$?
+$  if [ "$OK" -ne "0" ]; then echo "Verification failed!"; exit 1; done
+$  docker run lightninglabs/taproot-assets [command-line options]
+```
+
+# Signing an Existing Manifest File
+
+If you're a developer of `taproot-assets` and are interested in attaching your
+signature to the final release archive, the manifest MUST be signed in a manner
+that allows your signature to be verified by our verify script
+`scripts/verify-install.sh`. 
+
+Assuming you've done a local build for _all_ release targets, then you should
+have a file called `manifest-TAG.txt` where `TAG` is the actual release tag
+description being signed. The release script expects a particular file name for
+each included signature, so we'll need to modify the name of our output
+signature during signing.
+
+Assuming `USERNAME` is your current nick as a developer, then the following
+command will generate a proper signature:
+```shell
+$  gpg --detach-sig --output manifest-USERNAME-TAG.sig manifest-TAG.txt
+```

itest/loadtest/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21.0 as builder 
+FROM golang:1.21.0 as builder
 
 WORKDIR /app
 

make/builder.Dockerfile

@@ -1,9 +1,3 @@
-# If you change this value, please change it in the following files as well:
-# /.github/workflows/main.yaml
-# /Dockerfile
-# /dev.Dockerfile
-# /taprpc/Dockerfile
-# /tools/Dockerfile
 FROM golang:1.21.0-bookworm
 
 MAINTAINER Olaoluwa Osuntokun <[email protected]>

scripts/keys/ffranr.asc

@@ -0,0 +1,41 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQGNBGM+7DcBDACxPY5WZEB8PwmUkWugPTgDOiBtdOeh7V6B+emLAJjLMaj+XBIM
+YN4AmJOmhrTH3s2avmQRAmE2lYNDhAeYn6SGcHZonYXGMgnrbUpibYkUhsCz5n/T
+M2dG18rJEqVu7VjF7o9Ug8X5dmLgFTSfl4lne3bvR+ciuebWEZislpNRQs8FXBXk
+MQRDgJ44gjYyWcbdHOTimXM2vzDJo0OF2PHK1hJU23fHNH5Pnz7sjyCimEUbsO1f
+iVoSEp8tCGnFf20WWL39+Qsww96mr+Sm2UC746PKEVRDMIvkgG9VfzwbUP2oHvA0
+60gwm4QE2YIkfH7YZ4/pekRxkXoo6UobXMWS3S1iOQmJwR8sX3/51E96Fw98SIZx
+K7pOEQQ5ewkbLkr+rMhuP7noS5irXhqtcjfRLyLUrfP4Yj/OUlehiklULS2uQ3/h
+wpaBJJLw0Um8ARkR0hhk2gv8hhKbNm15sLwvtnDmRvCtzaxSxA8VkMcwS1OvxEGj
+IPwvLEb9MyBLvL0AEQEAAbQsUm9ieW4gRmZyYW5jb24gPHJvYnluQGxpZ2h0bmlu
+Zy5lbmdpbmVlcmluZz6JAdQEEwEKAD4WIQT+XhWacMQ21q9NKIex+ISFV6op0gUC
+Yz7sNwIbAwUJA8JnAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRCx+ISFV6op
+0jA4C/4tWj0/ByAHnguSgSSBuU4JTrfHngJP9Zle6cIGJkbt627on0PmRi59vPSu
+Zpgp+qwKk8hTZuOH4pvZMyBdGVrUsBjGO7DKU5ZaetR/aZxB4nTpc2nJ5nZXTiKV
+pn3MQ3wfnkq78DMQ7Qy3EO4D576bsJQVwiRoEAnrV4Wp3SQMss6jygZYi5dvpn1n
+/h8MI67XQDcwq1fkBZkRD6++GJGAkC3S0aPTytBhhUds+9Rh02JQV4S2Fy/niC5O
+6FIdmhwMZX79D5pxoWvk/9udVLJb9GK5wPwej9qtG0HcgI1ADmDbj1AXUd2iQy/I
+oo00Jy/iA3JHRudbeBM9cl9E1+azdE3c5AD55nrOeNekh4psfnkLC9RhP6I/Qa9v
+SQqLfwbp0hkuGsSn6PpsU+5P9NjUDida6aw5JERn/I3iGOe997EomCzrsKLlrLHa
+r1tfxutW+u0iS03sPFkiDRS4yD9VUyV/OmGxerXBRYrbZxVr0E7osYMyHvdtpNQU
+D1QMMsW5AY0EYz7sNwEMALyW6YZtvu94AJzEiL1LyT6wppW3OUdIjfWic0atFTKu
+4lWGg69SNmSS+qHbqYYB3tk8R1Vk1AmV0jT+jHI3nX7n6Sr65GAd6PsyWEG6KKbM
+Oz56eaA2MSzRxDqkM6mCwLk+OoERlns9KTtwHIz0rBL+9rgrUr9IXMPtW0RWgj9p
+T+qR+DsUt+QF54WZ4Fw5KrAmf4xIB6BYQ4Fx3RlufLomWgWmjfjwwS2YADY+YT8t
+y6ocRFH1LG0liVzbBR7VLCGizM2EyZU5Y7aTmLfnu52ldXSv7yeoHz+YNBljHpYZ
+N203/42/dW7+ybwlxratsDlYI+zPrbWvS+nSqDwPzpjWuIMbrZhm0fGl+Vitep6C
+WRPh+9fKVDSwxY5YlyU3AQz31zVQ3+n6Nog22Q9r23O4rYaY9BuO+XiWDntFZAC3
+xBcfJyEJ5mjSJUm7Rb7xxiW9pz1MgMAze+yoFA9qo2hK8IIHwoBTqODynpaeMisB
+733X9IIzDHk7xqFD0p0n/wARAQABiQG8BBgBCgAmFiEE/l4VmnDENtavTSiHsfiE
+hVeqKdIFAmM+7DcCGwwFCQPCZwAACgkQsfiEhVeqKdKjPwwAgWPyPKDJmDABWxKy
+NBRIUWYWX/HSqhF46VFXNVLpDlC7dbnxsng/82+qQRQWJ7haNsBM7qbhkkNXAl/g
+5R9etC21KYwiwH21i6p2HS0xj8xKuJRRaoj68eMhFFkr8+bIXtCaK4HY9/R0z1zD
+vkBlY1bo6JBFWAEGoGTBugqZiL6MiyK/jviQtWEWbHIysPBGfZray+i9zcdsl/t6
+NAHOVZp2mn6+UkrYQzgbpx2ix5APYA3/eutr/ajGSzgklEps7IOhWyLzwb6C5ujb
+UlK6UGK3vbZcEohmH7/iQpDHUt41fiq4VAbQnpL3Oihx9DFy/GnNDwvKc5GG568z
+Lqag3RLdZFJBM2T3bdgmk9LaFpM6gNfjh1D5XFfn+QS7tj2wvJ62CgVF7e2wjgn7
+iqXafTlM7vGN7a0Pb/aorRoEiEy3sQGEpFq6fSlHzVFa9AFwzrsdEeUDOnhOqlBM
+EdU8cqeZ7N0m8la8RU1kjOy03G+WmH3NY9X+mRafb877p+v9
+=bdL+
+-----END PGP PUBLIC KEY BLOCK-----