Add go.work, go.work.sum, and turn smartnode top level packages into modules

.github/workflows/build.yml

@@ -17,5 +17,4 @@ jobs:
       - uses: actions/setup-go@v4
         with:
           go-version: 1.21.8
-      - run: cd ${GITHUB_WORKSPACE}/rocketpool-cli && go build .
-      - run: cd ${GITHUB_WORKSPACE}/rocketpool && go build .
+      - run: make NO_DOCKER=true build/rocketpool-cli build/rocketpool-daemon

.github/workflows/lint.yml

@@ -13,29 +13,41 @@ permissions:
   # Optional: allow read access to pull request. Use with `only-new-issues` option.
   # pull-requests: read
 jobs:
-  golangci:
-    name: lint
+  detect-modules:
     runs-on: ubuntu-latest
+    outputs:
+      modules: ${{ steps.set-modules.outputs.modules }}
     steps:
-      - uses: actions/setup-go@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: 1.21.8
-      - uses: actions/checkout@v3
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v4
+      - id: set-modules
+        run: echo "modules=$(go list -m -json | jq -s '.' | jq -c '[.[].Dir]')" >> $GITHUB_OUTPUT
+  golangci-lint:
+    needs: detect-modules
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 1
+      matrix:
+        modules: ${{ fromJSON(needs.detect-modules.outputs.modules) }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: 1.21.8
+      - name: golangci-lint ${{ matrix.modules }}
+        uses: golangci/golangci-lint-action@v8
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
-          version: latest
+          version: v2.1
 
           # Optional: working directory, useful for monorepos
-          # working-directory: somedir
+          working-directory: ${{ matrix.modules }}
 
           # Optional: golangci-lint command line arguments.
           # args: --issues-exit-code=0
 
-          # For now, Smart Node will only enforce goimports linting
-          args: --disable-all --enable goimports
-
           # Optional: show only new issues if it's a pull request. The default value is `false`.
           # only-new-issues: true
 

.github/workflows/unit-tests.yml

@@ -17,4 +17,4 @@ jobs:
       - uses: actions/setup-go@v4
         with:
           go-version: 1.21.8
-      - run: go test ./... -timeout 30m
+      - run: make test

.golangci.yml

@@ -0,0 +1,13 @@
+version: "2"
+linters:
+  default: none 
+formatters:
+  enable:
+    - goimports
+run:
+  relative-path-mode: cfg
+  go: '1.21.8'
+output:
+  formats:
+    text:
+      path: stderr

Makefile

@@ -0,0 +1,135 @@
+VERSION=v$(shell cat shared/version.txt)
+LOCAL_OS=$(shell go env GOOS)-$(shell go env GOARCH)
+
+BUILD_DIR=build
+BIN_DIR=${BUILD_DIR}/${VERSION}/bin
+DOCKER_DIR=${BUILD_DIR}/${VERSION}/docker
+
+CLI_TARGET_OOS:=linux darwin
+ARCHS:=arm64 amd64
+
+CLI_TARGET_STRINGS:=$(foreach oos,$(CLI_TARGET_OOS), $(foreach arch,$(ARCHS),${BIN_DIR}/rocketpool-cli-$(oos)-$(arch)))
+DAEMON_TARGET_STRINGS:=$(foreach arch,$(ARCHS),${BIN_DIR}/rocketpool-daemon-linux-$(arch))
+
+MODULES:=$(foreach path,$(shell find . -name go.mod),$(dir $(path)))
+MODULE_GLOBS:=$(foreach module,$(MODULES),$(module)...)
+
+cli_deps = ${BIN_DIR}
+ifndef NO_DOCKER
+	cli_deps += docker-builder
+endif
+
+define rocketpool-cli-template
+.PHONY: ${BIN_DIR}/rocketpool-cli-$1-$2
+${BIN_DIR}/rocketpool-cli-$1-$2: ${cli_deps}
+	@echo "Building rocketpool-cli-$1-$2"
+ifndef NO_DOCKER
+	docker run --rm -v ./:/src --user $(shell id -u):$(shell id -g) -e CGO_ENABLED=0 \
+		-e GOARCH=$2 -e GOOS=$1 --workdir /src -v ~/.cache:/.cache rocketpool/smartnode-builder:${VERSION} \
+		go build -o $$@ rocketpool-cli/rocketpool-cli.go
+else
+	CGO_ENABLED=0 GOOS=$1 GOARCH=$2 go build -o $$@ ./rocketpool-cli/rocketpool-cli.go
+endif
+endef
+
+.PHONY: all
+all: ${BUILD_DIR}/rocketpool-cli ${BUILD_DIR}/rocketpool-daemon lint
+
+.PHONY: release
+release: ${CLI_TARGET_STRINGS} ${DAEMON_TARGET_STRINGS} ${BUILD_DIR}/rocketpool-cli ${BUILD_DIR}/rocketpool-daemon
+
+# Target for build/rocketpool-cli which is a symlink to an os-specific build
+${BUILD_DIR}/rocketpool-cli: ${BIN_DIR}/rocketpool-cli-${LOCAL_OS}
+	ln -sf $(shell pwd)/${BIN_DIR}/rocketpool-cli-${LOCAL_OS} ${BUILD_DIR}/rocketpool-cli
+
+
+# Target for build/rocketpool-daemon which is a symlink to an os-specific build
+${BUILD_DIR}/rocketpool-daemon: ${BIN_DIR}/rocketpool-daemon-${LOCAL_OS}
+	ln -sf $(shell pwd)/${BIN_DIR}/rocketpool-daemon-${LOCAL_OS} ${BUILD_DIR}/rocketpool-daemon
+
+# docker-builder container
+.PHONY: docker-builder
+docker-builder:
+	VERSION=${VERSION} docker bake -f docker/daemon-bake.hcl builder
+
+daemon_build_deps = ${BIN_DIR}
+ifndef NO_DOCKER
+	daemon_build_deps += docker-builder
+endif
+
+# amd64 daemon build
+.PHONY: ${BIN_DIR}/rocketpool-daemon-linux-amd64
+${BIN_DIR}/rocketpool-daemon-linux-amd64: ${daemon_build_deps}
+ifndef NO_DOCKER
+	docker run --rm -v ./:/src --user $(shell id -u):$(shell id -g) -e CGO_ENABLED=1 -e CGO_C_FLAGS="-O -D__BLST_PORTABLE__" \
+		-e GOARCH=amd64 -e GOOS=linux --workdir /src -v ~/.cache:/.cache rocketpool/smartnode-builder:${VERSION} \
+		go build -o $@ rocketpool/rocketpool.go
+else
+	CGO_ENABLED=1 CGO_C_FLAGS="-O -D__BLST_PORTABLE__" GOARCH=amd64 GOOS=linux go build -o $@ rocketpool/rocketpool.go
+endif
+
+# arm64 daemon build
+.PHONY: ${BIN_DIR}/rocketpool-daemon-linux-arm64
+${BIN_DIR}/rocketpool-daemon-linux-arm64: ${daemon_build_deps}
+ifndef NO_DOCKER
+	docker run --rm -v ./:/src --user $(shell id -u):$(shell id -g) -e CGO_ENABLED=1 -e CGO_C_FLAGS="-O -D__BLST_PORTABLE__" \
+	-e CC=aarch64-linux-gnu-gcc -e CXX=aarch64-linux-gnu-cpp -e CGO_C_FLAGS="-O -D__BLST_PORTABLE__" -e GOARCH=arm64 -e GOOS=linux \
+	--workdir /src -v ~/.cache:/.cache rocketpool/smartnode-builder:${VERSION} \
+	go build -o $@ rocketpool/rocketpool.go
+else
+	CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc CXX=aarch64-linux-gnu-cpp CGO_C_FLAGS="-O -D__BLST_PORTABLE__" GOARCH=arm64 GOOS=linux go build -o $@ rocketpool/rocketpool.go
+endif
+
+${BIN_DIR}:
+	mkdir -p ${BIN_DIR}
+${DOCKER_DIR}:
+	mkdir -p ${DOCKER_DIR}
+
+$(foreach oos,$(CLI_TARGET_OOS),$(foreach arch,$(ARCHS),$(eval $(call rocketpool-cli-template,$(oos),$(arch)))))
+
+
+# Docker containers
+.PHONY: docker
+docker: ${DOCKER_DIR}
+	VERSION=${VERSION} docker bake -f docker/daemon-bake.hcl daemon
+
+.PHONY: docker-load
+docker-load: docker
+	docker import - smartnode:${VERSION}-amd64 < ${DOCKER_DIR}/smartnode:${VERSION}-amd64.tar
+	docker import - smartnode:${VERSION}-arm64 < ${DOCKER_DIR}/smartnode:${VERSION}-arm64.tar
+
+.PHONY: docker-push
+docker-push: docker-load
+	echo
+	echo -n "Publishing smartnode:${VERSION} containers. Continue? [yN]: " && read ans && if [ $${ans:-'N'} != 'y' ]; then exit 1; fi
+	docker push rocketpool/smartnode:${VERSION}-amd64
+	docker push rocketpool/smartnode:${VERSION}-arm64
+	echo "Done!"
+
+.PHONY: docker-latest
+docker-latest: docker-push
+	echo
+	echo -n "Publishing smartnode:${VERSION} as latest. Continue? [yN]: " && read ans && if [ $${ans:-'N'} != 'y' ]; then exit 1; fi
+	rm -rf ~/.docker/manifests/docker.io_rocketpool_smartnode-latest
+	rm -rf ~/.docker/manifests/docker.io_rocketpool_smartnode-${VERSION}
+	docker manifest create rocketpool/smartnode:${VERSION} --amend rocketpool/smartnode:${VERSION}-amd64 --amend rocketpool/smartnode:${VERSION}-arm64
+	docker manifest create rocketpool/smartnode:latest --amend rocketpool/smartnode:${VERSION}-amd64 --amend rocketpool/smartnode:${VERSION}-arm64
+	docker manifest push --purge rocketpool/smartnode:${VERSION}
+	docker manifest push --purge rocketpool/smartnode:latest
+
+define lint-template 
+.PHONY: lint-$1
+lint-$1:
+	docker run -e GOCACHE=/go/.cache/go-build -e GOLANGCI_LINT_CACHE=/go/.cache/golangci-lint --user $(shell id -u):$(shell id -g) --rm -v ~/.cache:/go/.cache -v .:/smartnode --workdir /smartnode/$1 golangci/golangci-lint:v2.1-alpine golangci-lint fmt --diff
+endef
+$(foreach module,$(MODULES),$(eval $(call lint-template,$(module))))
+.PHONY: lint
+lint: $(foreach module,$(MODULES),lint-$(module))
+
+.PHONY: test
+test:
+	go test -test.timeout 20m $(MODULE_GLOBS)
+
+.PHONY: clean
+clean:
+	rm -rf ${BUILD_DIR}

README.md

@@ -24,6 +24,26 @@ See the [Rocket Pool dockerhub](https://hub.docker.com/u/rocketpool) page for a
 
 See the [Smartnode Installer](https://github.com/rocket-pool/smartnode-install) repository for supported platforms and installation instructions.
 
+## Development
+
+A [Makefile](./Makefile) is included for building, testing, and linting.
+
+* `make` or `make all` will build rocketpool-cli, rocketpool-daemon, and run the linter.
+  * build/rocketpool-cli and build/rocketpool-daemon will be symlinked to the version and architecture specific binaries in build/
+* `make release` will build all architecture specific binaries as well as docker images and manifests
+  * It will tag docker images as latest as well as the version in `shared/version.txt`
+  * It will put cli and native mode binaries in build/\<version\>
+* `make build/rocketpool-cli` builds just the cli
+  * The cli is always built natively.
+* `make build/rocketpool-daemon` builds just the daemon
+  * The daemon is built in docker, unless you run `make NO_DOCKER=true \<cmd\>`
+* `make docker` builds the rocketpool/smartnode containers for all supported architectures and saves them in build/\<version\>/docker
+* `make docker-load` builds and loads the smartnode containers.
+* `make docker-push` builds, loads, and pushes the smartnode containers.
+* `make docker-latest` builds, loads, pushes, tags as latest, and creates a multi-arch manifest, which is also pushed.
+* `make lint` runs the linter.
+* `make test` runs all unit tests.
+* `make clean` deletes any binaries. It does not clear your go caches. It does not clean up old docker images.
 
 ## CLI Commands
 

addons/go.mod

@@ -0,0 +1,14 @@
+module github.com/rocket-pool/smartnode/addons
+
+go 1.21
+
+require (
+	github.com/ethereum/go-ethereum v1.13.5
+	google.golang.org/protobuf v1.32.0
+)
+
+require (
+	github.com/google/go-cmp v0.6.0 // indirect
+	golang.org/x/crypto v0.19.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
+)

docker/daemon-bake.hcl

@@ -0,0 +1,35 @@
+variable "VERSION" {
+  default = "$VERSION"
+}
+
+group "default" {
+  targets = ["builder", "daemon"]
+}
+
+target "builder" {
+  dockerfile = "docker/rocketpool-dockerfile"
+  tags = [ 
+    "rocketpool/smartnode-builder:${VERSION}",
+    "rocketpool/smartnode-builder:local"
+  ]
+  target = "smartnode_dependencies"
+  platforms = [ "linux/amd64" ]
+}
+
+target "daemon" {
+  name = "daemon-${arch}"
+  dockerfile = "docker/rocketpool-dockerfile"
+  args = {
+    VERSION = "${VERSION}"
+  }
+  tags = [
+    "rocketpool/smartnode:${VERSION}-${arch}",
+    "localhost/rocketpool/smartnode:${VERSION}-${arch}"
+  ]
+  matrix = {
+    arch = [ "amd64", "arm64" ]
+  }
+  target = "daemon"
+  platform = "linux/${arch}"
+  output = [{ "type": "tar", "dest": "build/${VERSION}/docker/smartnode:${VERSION}-${arch}.tar" }]
+}

docker/rocketpool-dockerfile

@@ -1,7 +1,44 @@
-FROM debian:bookworm-slim
+FROM golang:1.21.8-bookworm AS smartnode_dependencies
 
+# Install build tools
+RUN dpkg --add-architecture arm64
+RUN apt update && apt install -y \
+	build-essential \
+	gcc-aarch64-linux-gnu \
+	libc6-dev-arm64-cross\
+	g++-aarch64-linux-gnu \
+	wget
+
+# Cache go dependencies
+COPY ./go.work /src/go.work
+COPY ./go.work.sum /src/go.work.sum
+COPY ./addons/go.mod /src/addons/go.mod
+COPY ./rocketpool/go.mod /src/rocketpool/go.mod
+COPY ./rocketpool-cli/go.mod /src/rocketpool-cli/go.mod
+COPY ./shared/go.mod /src/shared/go.mod
+WORKDIR /src
+RUN go mod download -x
+WORKDIR /
+RUN rm -rf /src
+
+FROM smartnode_dependencies AS build
 ARG TARGETARCH
-COPY ./rocketpool/rocketpool-daemon-linux-${TARGETARCH} /go/bin/rocketpool
+ARG VERSION
+COPY ./go.work /src/go.work
+COPY ./go.work.sum /src/go.work.sum
+COPY ./addons/ /src/addons/
+COPY ./rocketpool/ /src/rocketpool/
+COPY ./rocketpool-cli/ /src/rocketpool-cli/
+COPY ./shared/ /src/shared/
+COPY Makefile /src/Makefile
+WORKDIR /src
+RUN --mount=type=cache,target=/root/.cache/go-build make NO_DOCKER=true build/${VERSION}/bin/rocketpool-daemon-linux-${TARGETARCH}
+
+FROM debian:bookworm-slim AS daemon
+ARG TARGETARCH
+ARG VERSION
+
+COPY --from=build /src/build/${VERSION}/bin/rocketpool-daemon-linux-${TARGETARCH} /go/bin/rocketpool
 
 RUN apt update && apt install ca-certificates -y