Build refactor (Start9Labs#116)

* Add option for Ordinals Spam Filter
* update alpine base image to 3.16
* old GPG keys cleanup
* update key server
* use ssl when installing apk's for finall image
* remove apk cache after install

* Security update for build dependencies

* Automate submodule update handling with make

* Build refactor

* make arm and x86 build targets update

Author: k0gen

.gitignore

@@ -1,7 +1,7 @@
 manager/target/
 **/*.rs.bk
 bitcoind.s9pk
-image.tar
+.DS_Store
 .vscode/
 scripts/embassy.js
 docker-images/

Dockerfile

@@ -4,7 +4,7 @@
 FROM lncm/berkeleydb as berkeleydb
 
 # Build stage for Bitcoin Core
-FROM alpine:3.12 as bitcoin-core
+FROM alpine:3.16 as bitcoin-core
 
 COPY --from=berkeleydb /opt /opt
 
@@ -24,71 +24,22 @@ RUN apk --no-cache add sqlite-dev
 RUN apk --no-cache add zeromq-dev
 RUN set -ex \
   && for key in \
-9D3CC86A72F8494342EA5FD10A41BDC3F4FAFF1C \
-617C90010B3BD370B0AC7D424BB42E31C79111B8 \
-E944AE667CF960B1004BC32FCA662BE18B877A60 \
-152812300785C96444D3334D17565732E08E5E41 \
-590B7292695AFFA5B672CBB2E13FC145CD3F4304 \
-0AD83877C1F0CD1EE9BD660AD7CC770B81FD22A8 \
-912FD3228387123DC97E0E57D5566241A0295FA9 \
-04017A2A6D9A0CCDC81D8EC296AB007F1A7ED999 \
-C519EBCF3B926298946783EFF6430754120EC2F4 \
-18AE2F798E0D239755DA4FD24B79F986CBDF8736 \
-101598DC823C1B5F9A6624ABA5E0907A0380E6C3 \
-F20F56EF6A067F70E8A5C99FFF95FAA971697405 \
-C060A6635913D98A3587D7DB1C2491FFEB0EF770 \
-BF6273FAEF7CC0BA1F562E50989F6B3048A116B5 \
-6D3170C1DC2C6FD0AEEBCA6743811D1A26623924 \
-948444FCE03B05BA5AB0591EC37B1C1D44C786EE \
-1C6621605EC50319C463D56C7F81D87985D61612 \
-9A1689B60D1B3CCE9262307A2F40A9BF167FBA47 \
-D35176BE9264832E4ACA8986BF0792FBE95DC863 \
-6F993B250557E7B016ADE5713BDCDA2D87A881D9 \
-01CDF4627A3B88AAE4A571C87588242FBE38D3A8 \
-D1DBF2C4B96F2DEBF4C16654410108112E7EA81F \
-2688F5A9A4BE0F295E921E8A25F27A38A47AD566 \
-D3F22A3A4C366C2DCB66D3722DA9C5A7FA81EA35 \
-7480909378D544EA6B6DCEB7535B12980BB8A4D3 \
-D3CC177286005BB8FF673294C5242A1AB3936517 \
-82921A4B88FD454B7EB8CE3C796C4109063D4EAF \
-32EE5C4C3FA15CCADB46ABE529D4BCB6416F53EC \
-4B4E840451149DD7FB0D633477DFAB5C3108B9A8 \
-C42AFF7C61B3E44A1454CD3557AF762DB3353322 \
-70A1D47DD44F59DF8B22244333E472FE870C7E5D \
-30DE693AE0DE9E37B3E7EB6BBFF0F67810C1EED1 \
-E463A93F5F3117EEDE6C7316BD02942421F4889F \
-B8B3F1C0E58C15DB6A81D30C3648A882F4316B9B \
-07DF3E57A548CCFB7530709189BBB8663E2E65CE \
-CA03882CB1FC067B5D3ACFE4D300116E1C875A3D \
-E777299FC265DD04793070EB944D35F9AC3DB76A \
-AD5764F4ADCE1B99BDFD179E12335A271D4D62EC \
-9692B91BBF0E8D34DFD33B1882C5C009628ECF0C \
-C57E4B42223FDE851D4F69DD28DF2724F241D8EE \
-F4FC70F07310028424EFC20A8E4256593F177720 \
-D62A803E27E7F43486035ADBBCD04D8E9CCCAC2A \
-37EC7D7B0A217CDB4B4E007E7FAB114267E4FA04 \
-D762373D24904A3E42F33B08B9A408E71DAAC974 \
-133EAC179436F14A5CF1B794860FEB804E669320 \
-6A8F9C266528E25AEB1D7731C2371D91CB716EA7 \
-A8FC55F3B04BA3146F3492E79303B33A305224CB \
-ED9BDF7AD6A55E232E84524257FF9BDBCC301009 \
-867345026B6763E8B07EE73AB6737117397F5C4F \
-9EDAFF80E080659604F4A76B2EBB056FD847F8A7 \
-6DEEF79B050C4072509B743F8C275BC595448867 \
-AEC1884398647C47413C1C3FB1179EB7347DC10D \
-74E2DEF5D77260B98BC19438099BAD163C70FBFA \
-79D00BAC68B56D422F945A8F8E3A8F3247DBCBBF \
-71A3B16735405025D447E8F274810B012346C9A6 \
-287AE4CA1187C68C08B49CB2D11BD4F33F1DB499 \
-F9A8737BF4FF5C89C903DF31DD78544CF91B1514 \
-4DAF18FE948E7A965B30F9457E296D555E7F63A7 \
-28E72909F1717FE9607754F8A7BEB2621678D37D \
-F19F5FF2B0589EC341220045BA03F4DBE0C63FB4 \
+  152812300785C96444D3334D17565732E08E5E41 \
+  0AD83877C1F0CD1EE9BD660AD7CC770B81FD22A8 \
+  590B7292695AFFA5B672CBB2E13FC145CD3F4304 \
+  CFB16E21C950F67FA95E558F2EEB9F5CC09526C1 \
+  F4FC70F07310028424EFC20A8E4256593F177720 \
+  D1DBF2C4B96F2DEBF4C16654410108112E7EA81F \
+  287AE4CA1187C68C08B49CB2D11BD4F33F1DB499 \
+  9DEAE0DC7063249FB05474681E4AED62986CD25D \
+  3EB0DEE6004A13BE5A0CC758BF2978B068054311 \
+  ED9BDF7AD6A55E232E84524257FF9BDBCC301009 \
+  28E72909F1717FE9607754F8A7BEB2621678D37D \
+  79D00BAC68B56D422F945A8F8E3A8F3247DBCBBF \
   ; do \
-  gpg --batch --keyserver hkps://keyserver.ubuntu.com --recv-keys "$key" || \
-  gpg --batch --keyserver hkps://pgp.mit.edu --recv-keys "$key" || \
+  gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key" || \
   gpg --batch --keyserver keyserver.pgp.com --recv-keys "$key" || \
-  gpg --batch --keyserver hkps://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \
+  gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys "$key" || \
   gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \
   gpg --batch --keyserver keys.openpgp.org --recv-keys "$key" ; \
   done
@@ -100,7 +51,7 @@ ENV BITCOIN_PREFIX=/opt/bitcoin-${BITCOIN_VERSION}
 RUN wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/SHA256SUMS
 RUN wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/SHA256SUMS.asc
 RUN wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/bitcoin-${BITCOIN_VERSION}.tar.gz
-RUN gpg --verify SHA256SUMS.asc
+RUN gpg --verify SHA256SUMS.asc SHA256SUMS
 RUN grep " bitcoin-${BITCOIN_VERSION}.tar.gz\$" SHA256SUMS | sha256sum -c -
 RUN tar -xzf *.tar.gz
 
@@ -127,29 +78,28 @@ RUN strip ${BITCOIN_PREFIX}/lib/libbitcoinconsensus.a
 RUN strip ${BITCOIN_PREFIX}/lib/libbitcoinconsensus.so.0.0.0
 
 # Build stage for compiled artifacts
-FROM alpine:3.13
+FROM alpine:3.16
 
 LABEL maintainer.0="João Fonseca (@joaopaulofonseca)" \
   maintainer.1="Pedro Branco (@pedrobranco)" \
   maintainer.2="Rui Marinho (@ruimarinho)" \
   maintainer.3="Aiden McClelland (@dr-bonez)"
 
-RUN apk update
+RUN sed -i 's/http\:\/\/dl-cdn.alpinelinux.org/https\:\/\/alpine.global.ssl.fastly.net/g' /etc/apk/repositories
 RUN apk --no-cache add \
   bash \
   curl \
-  boost-filesystem=1.72.0-r6 \
-  boost-system=1.72.0-r6 \
-  boost-thread=1.72.0-r6 \
+  boost-filesystem \
+  boost-system \
+  boost-thread \
   libevent \
   libzmq \
-  sqlite-libs \
-  su-exec \
-  tini
+  sqlite-dev \
+  tini\
+  yq
+RUN rm -rf /var/cache/apk/*
 
-ARG PLATFORM
 ARG ARCH
-RUN wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_${PLATFORM} && chmod +x /usr/local/bin/yq
 
 ENV BITCOIN_DATA=/root/.bitcoin
 ARG BITCOIN_VERSION

Makefile

@@ -5,43 +5,72 @@ VERSION_CORE := "24.0.1"
 
 .DELETE_ON_ERROR:
 
-all: verify
+all: submodule-update verify
 
 clean:
 	rm -f $(PKG_ID).s9pk
 	rm -f docker-images/*.tar
 	rm -f scripts/*.js
 
 verify: $(PKG_ID).s9pk
-	embassy-sdk verify s9pk $(PKG_ID).s9pk
+	@embassy-sdk verify s9pk $(PKG_ID).s9pk
+	@echo " Done!"
+	@echo "   Filesize: $(shell du -h $(PKG_ID).s9pk) is ready"
 
-# for rebuilding just the arm image. will include docker-images/x86_64.tar into the s9pk if it exists
-arm: docker-images/aarch64.tar scripts/embassy.js
-	embassy-sdk pack
+# for rebuilding just the arm image.
+arm:
+	@rm -f docker-images/x86_64.tar
+	ARCH=aarch64 $(MAKE)
 
-# for rebuilding just the x86 image. will include docker-images/aarch64.tar into the s9pk if it exists
-x86: docker-images/x86_64.tar scripts/embassy.js
-	embassy-sdk pack
+# for rebuilding just the x86 image.
+x86:
+	@rm -f docker-images/aarch64.tar
+	ARCH=x86_64 $(MAKE)
 
 $(PKG_ID).s9pk: manifest.yaml assets/compat/* docker-images/aarch64.tar docker-images/x86_64.tar instructions.md scripts/embassy.js
-	embassy-sdk pack
+ifeq ($(ARCH),aarch64)
+	@echo "embassy-sdk: Preparing aarch64 package ..."
+else ifeq ($(ARCH),x86_64)
+	@echo "embassy-sdk: Preparing x86_64 package ..."
+else
+	@echo "embassy-sdk: Preparing Universal Package ..."
+endif
+	@embassy-sdk pack
 
 install: $(PKG_ID).s9pk
+ifeq (,$(wildcard ~/.embassy/config.yaml))
+	@echo; echo "You must define \"host: http://embassy-server-name.local\" in ~/.embassy/config.yaml config file first"; echo
+else
 	embassy-cli package install $(PKG_ID).s9pk
+endif
 
 docker-images/aarch64.tar: Dockerfile docker_entrypoint.sh manager/target/aarch64-unknown-linux-musl/release/bitcoind-manager manifest.yaml check-rpc.sh check-synced.sh migrations/* actions/*
+ifeq ($(ARCH),x86_64)
+else
 	mkdir -p docker-images
 	docker buildx build --tag start9/$(PKG_ID)/main:$(PKG_VERSION) --build-arg BITCOIN_VERSION=$(VERSION_CORE) --build-arg ARCH=aarch64 --build-arg PLATFORM=arm64 --platform=linux/arm64 -o type=docker,dest=docker-images/aarch64.tar .
+endif
 
 docker-images/x86_64.tar: Dockerfile docker_entrypoint.sh manager/target/x86_64-unknown-linux-musl/release/bitcoind-manager manifest.yaml check-rpc.sh check-synced.sh migrations/* actions/*
+ifeq ($(ARCH),aarch64)
+else
 	mkdir -p docker-images
 	docker buildx build --tag start9/$(PKG_ID)/main:$(PKG_VERSION) --build-arg BITCOIN_VERSION=$(VERSION_CORE) --build-arg ARCH=x86_64 --build-arg PLATFORM=amd64 --platform=linux/amd64 -o type=docker,dest=docker-images/x86_64.tar .
+endif
 
 manager/target/aarch64-unknown-linux-musl/release/bitcoind-manager: $(MANAGER_SRC)
-	docker run --rm -it -v ~/.cargo/registry:/root/.cargo/registry -v "$(shell pwd)"/manager:/home/rust/src start9/rust-musl-cross:aarch64-musl cargo build --release
+	docker run --rm -it -v ~/.cargo/registry:/root/.cargo/registry -v "$(shell pwd)"/manager:/home/rust/src messense/rust-musl-cross:aarch64-musl cargo build --release
 
 manager/target/x86_64-unknown-linux-musl/release/bitcoind-manager: $(MANAGER_SRC)
-	docker run --rm -it -v ~/.cargo/registry:/root/.cargo/registry -v "$(shell pwd)"/manager:/home/rust/src start9/rust-musl-cross:x86_64-musl cargo build --release
+	docker run --rm -it -v ~/.cargo/registry:/root/.cargo/registry -v "$(shell pwd)"/manager:/home/rust/src messense/rust-musl-cross:x86_64-musl cargo build --release
 
 scripts/embassy.js: scripts/**/*.ts
 	deno bundle scripts/embassy.ts scripts/embassy.js
+
+submodule-update:
+	@if [ -z "$(shell git submodule status | egrep -v '^ '|awk '{print $$2}')" ]; then \
+		echo "Submodules are up to date."; \
+	else \
+		echo "\nUpdating submodules...\n"; \
+		git submodule update --init --progress; \
+	fi

bitcoin

@@ -2,10 +2,9 @@
 
 set -euo pipefail
 
+CONFIG_FILE="/root/.bitcoin/start9/config.yaml"
 export EMBASSY_IP=$(ip -4 route list match 0/0 | awk '{print $3}')
-export PEER_TOR_ADDRESS=$(yq e '.peer-tor-address' /root/.bitcoin/start9/config.yaml)
-export RPC_TOR_ADDRESS=$(yq e '.rpc-tor-address' /root/.bitcoin/start9/config.yaml)
+export PEER_TOR_ADDRESS=$(yq e '.peer-tor-address' "$CONFIG_FILE")
+export RPC_TOR_ADDRESS=$(yq e '.rpc-tor-address' "$CONFIG_FILE")
 
-# lighttpd -f /root/.bitcoin/httpd.conf
-# exec /usr/local/bin/bitcoind-manager
 exec tini -p SIGTERM -- bitcoind-manager