Base. Added alternative snapshot downloads from S3 URL.

Author: vlasonfa

lib/base/lib/assets/restore-from-snapshot-archive-s3.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+set +e
+
+source /etc/environment
+
+echo "Downloading Snapshot."
+
+cd /data
+
+SNAPSHOT_FILE_NAME=snapshot.tar.gz
+SNAPSHOT_DIR=/data
+
+LATEST_SNAPSHOT_FILE_NAME=$(curl https://$NETWORK_ID-$NODE_CONFIG-snapshots.base.org/latest) && \
+s5cmd --log error cp s3://base-snapshots-$NETWORK_ID-archive/$LATEST_SNAPSHOT_FILE_NAME /data && \
+echo "Downloading Snapshot script finished" && \
+sleep 60 &&\
+echo "Starting snapshot decompression ..." && \
+tar -zxvf  $SNAPSHOT_DIR/$SNAPSHOT_FILE_NAME -C /data 2>&1 | tee unzip.log && echo "decompresed successfully..." || echo "decompression failed..." >> snapshots-decompression.log
+
+echo "Decompresed snapshot, cleaning up..."
+
+mv /data/snapshots/$NETWORK_ID/download/* /data && \
+rm -rf /data/snapshots && \
+rm -rf $SNAPSHOT_DIR/$SNAPSHOT_FILE_NAME
+
+echo "Snapshot is ready, starting the service.."
+
+chown -R bcuser:bcuser /data
+
+sudo systemctl daemon-reload
+sudo systemctl enable --now base

lib/base/lib/assets/restore-from-snapshot.sh renamed to lib/base/lib/assets/restore-from-snapshot-http.sh

@@ -16,7 +16,6 @@ if [ "$SNAPSHOT_URL" == "none" ] || [ -z "${SNAPSHOT_URL}" ]; then
   SNAPSHOT_URL=https://$NETWORK_ID-$NODE_CONFIG-snapshots.base.org/$LATEST_SNAPSHOT_FILE_NAME
 fi
 
-# take about 1 hour to download the Snapshot
 while (( SNAPSHOT_DOWNLOAD_STATUS != 0 ))
 do
         PIDS=$(pgrep aria2c)

lib/base/lib/assets/setup-instance-store-volumes.sh

@@ -13,8 +13,13 @@ if [ -n "$DATA_VOLUME_ID" ]; then
 
     mkfs.ext4 $DATA_VOLUME_ID
     echo "Data volume formatted. Mounting..."
-    # Waiting wihtouht using sleep as it sometimes just hangs....
-    coproc read -t 10 && wait "$!" || true
+    echo "waiting for volume to get UUID"
+    OUTPUT=0;
+    while [ "$OUTPUT" = 0 ]; do 
+      DATA_VOLUME_UUID=$(lsblk -fn -o UUID $DATA_VOLUME_ID)
+      OUTPUT=$(echo $DATA_VOLUME_UUID | grep -c - $2)
+      echo $OUTPUT
+    done
     DATA_VOLUME_UUID=$(lsblk -fn -o UUID  $DATA_VOLUME_ID)
     DATA_VOLUME_FSTAB_CONF="UUID=$DATA_VOLUME_UUID /data ext4 defaults 0 2"
     echo "DATA_VOLUME_ID="$DATA_VOLUME_ID

lib/base/lib/assets/user-data/node.sh

@@ -35,7 +35,7 @@ yum update -y
 yum -y install amazon-cloudwatch-agent collectd jq yq gcc ncurses-devel aws-cfn-bootstrap zstd
 wget $YQ_URI -O /usr/bin/yq && chmod +x /usr/bin/yq
 
-# install aria2 a p2p downloader
+echo " Installing aria2 a p2p downloader"
 cd /tmp
 
 if [ "$arch" == "x86_64" ]; then
@@ -50,6 +50,14 @@ else
   make install
 fi
 
+echo " Installing s5cmd"
+cd /opt
+wget -q $S5CMD_URI -O s5cmd.tar.gz
+tar -xf s5cmd.tar.gz
+chmod +x s5cmd
+mv s5cmd /usr/bin
+s5cmd version
+
 cd /opt
 
 echo "Downloading assets zip file"
@@ -83,7 +91,6 @@ yum install -y $SSM_AGENT_BINARY_URI
 REGION=${_REGION_}
 STACK_NAME=${_STACK_NAME_}
 RESTORE_FROM_SNAPSHOT=${_RESTORE_FROM_SNAPSHOT_}
-FORMAT_DISK=${_FORMAT_DISK_}
 NETWORK_ID=${_NETWORK_ID_}
 NODE_CONFIG=${_NODE_CONFIG_}
 L1_EXECUTION_ENDPOINT=${_L1_EXECUTION_ENDPOINT_}
@@ -290,9 +297,15 @@ if [ "$RESTORE_FROM_SNAPSHOT" == "false" ]; then
   systemctl daemon-reload
   systemctl enable --now base
 else
-  echo "Restoring node from snapshot"
-  chmod +x /opt/restore-from-snapshot.sh
-  echo "/opt/restore-from-snapshot.sh" | at now + 1 min
+  if [ "$NODE_CONFIG" == "archive" ]; then
+    echo "Restoring archive node from snapshot over s3"
+    chmod +x /opt/restore-from-snapshot-archive-s3.sh
+    echo "/opt/restore-from-snapshot-archive-s3.sh" | at now + 1 min
+  else 
+    echo "Restoring full node from snapshot over http"
+    chmod +x /opt/restore-from-snapshot-http.sh
+    echo "/opt/restore-from-snapshot-http.sh" | at now + 1 min
+  fi
 fi
 
 if [[ "$LIFECYCLE_HOOK_NAME" != "none" ]]; then

lib/base/single-archive-node-deploy.json

@@ -1,5 +1,5 @@
 {
   "base-single-node-archive-mainnet": {
-    "nodeinstanceid": "i-0206eadc184b36db5"
+    "nodeinstanceid": "i-03efb8b36f80c2fd1"
   }
 }