Base. Debugging deployments and snapshot download

vlasonfa · vlasonfa · commit 0ea2ea5a2430 · 2024-05-06T15:28:25.000+10:00
diff --git a/lib/base/app.ts b/lib/base/app.ts
@@ -15,14 +15,16 @@ new BaseCommonStack(app, "base-common", {
 });
 
 new BaseSingleNodeStack(app, "base-single-node", {
-  stackName: `base-single-node-${config.baseNodeConfig.baseNetworkId}`,
+  stackName: `base-single-node-${config.baseNodeConfig.baseNodeConfiguration}-${config.baseNodeConfig.baseNetworkId}`,
   env: { account: config.baseConfig.accountId, region: config.baseConfig.region },
 
   instanceType: config.baseNodeConfig.instanceType,
   instanceCpuType: config.baseNodeConfig.instanceCpuType,
   baseNetworkId: config.baseNodeConfig.baseNetworkId,
+  baseNodeConfiguration: config.baseNodeConfig.baseNodeConfiguration,
   restoreFromSnapshot: config.baseNodeConfig.restoreFromSnapshot,
   l1ExecutionEndpoint: config.baseNodeConfig.l1ExecutionEndpoint,
   l1ConsensusEndpoint: config.baseNodeConfig.l1ConsensusEndpoint,
+  snapshotUrl: config.baseNodeConfig.snapshotUrl,
   dataVolume: config.baseNodeConfig.dataVolume,
 });
diff --git a/lib/base/lib/assets/restore-from-snapshot.sh b/lib/base/lib/assets/restore-from-snapshot.sh
diff --git a/lib/base/lib/assets/start-from-snapshot.sh b/lib/base/lib/assets/start-from-snapshot.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+
+source /etc/environment
+echo "Downloading snpashot"
+
+cd /data
+
+BASE_SNAPSHOT_FILE_NAME=snalshot.tar.gz
+BASE_SNAPSHOT_DIR=/data/
+BASE_SNAPSHOT_DOWNLOAD_STATUS=-1
+BASE_LATEST_SNAPSHOT_FILE_NAME=$(curl https://$NETWORK_ID-$NODE_CONFIG-snapshots.base.org/latest)
+
+if [ "$SNAPSHOT_URL" == "none" ] || [ -z "${SNAPSHOT_URL}" ]; then
+  SNAPSHOT_URL=https://$NETWORK_ID-$NODE_CONFIG-snapshots.base.org/$BASE_LATEST_SNAPSHOT_FILE_NAME
+fi
+
+while (( BASE_SNAPSHOT_DOWNLOAD_STATUS != 0 ))
+do
+        PIDS=$(pgrep aria2c)
+        if [ -z "$PIDS" ]; then
+                aria2c -x3 $SNAPSHOT_URL -d $BASE_SNAPSHOT_DIR -o $BASE_SNAPSHOT_FILE_NAME -l aria2c.log --log-level=warn --allow-piece-length-change=true
+        fi
+        BASE_SNAPSHOT_DOWNLOAD_STATUS=$?
+        pid=$(pidof aria2c)
+        wait $pid
+        echo "aria2c exit."
+        case $BASE_SNAPSHOT_DOWNLOAD_STATUS in
+                3)
+                        echo "file not exist."
+                        exit 3
+                        ;;
+                9)
+                        echo "No space left on device."
+                        exit 9
+                        ;;
+                *)
+                        continue
+                        ;;
+        esac
+done
+
+echo "Downloading snapshot succeed"
+
+sleep 60
+# take about 2 hours to decompress the snapshot
+echo "Decompression snapshot start ..."
+
+tar -I zstdmt -xf  $BASE_SNAPSHOT_DIR/$BASE_SNAPSHOT_FILE_NAME -C /data 2>&1 | tee unzip.log && echo "decompression success..." || echo "decompression failed..." >> snapshots-decompression.log
+echo "Decompressing snapshot success ..."
+
+mv /data/snapshots/$NETWORK_ID/download/* /data && \
+rm -rf /data/snapshots && \
+rm -rf /data/$BASE_SNAPSHOT_FILE_NAME
+
+echo "Snapshot is ready !!!"
+
+chown -R bcuser:bcuser /data && \
+sudo su bcuser && \
+/usr/local/bin/docker-compose -f /home/bcuser/node/docker-compose.yml up -d
diff --git a/lib/base/lib/assets/user-data/node.sh b/lib/base/lib/assets/user-data/node.sh
@@ -6,9 +6,11 @@ LIFECYCLE_HOOK_NAME=${_LIFECYCLE_HOOK_NAME_}
 AUTOSCALING_GROUP_NAME=${_AUTOSCALING_GROUP_NAME_}
 RESOURCE_ID=${_NODE_CF_LOGICAL_ID_}
 ASSETS_S3_PATH=${_ASSETS_S3_PATH_}
-echo "LIFECYCLE_HOOK_NAME=$LIFECYCLE_HOOK_NAME" >> /etc/environment
-echo "AUTOSCALING_GROUP_NAME=$AUTOSCALING_GROUP_NAME" >> /etc/environment
-echo "ASSETS_S3_PATH=$ASSETS_S3_PATH" >> /etc/environment
+{
+  echo "LIFECYCLE_HOOK_NAME=$LIFECYCLE_HOOK_NAME"
+  echo "AUTOSCALING_GROUP_NAME=$AUTOSCALING_GROUP_NAME"
+  echo "ASSETS_S3_PATH=$ASSETS_S3_PATH"
+} >> /etc/environment
 
 arch=$(uname -m)
 
@@ -55,19 +57,22 @@ unzip -q awscliv2.zip
 rm /usr/bin/aws
 ln /usr/local/bin/aws /usr/bin/aws
 
-aws configure set default.s3.max_concurrent_requests 50
-aws configure set default.s3.multipart_chunksize 256MB
-
 echo 'Installing SSM Agent'
 yum install -y $SSM_AGENT_BINARY_URI
 
-echo "Installing s5cmd"
-cd /opt
-wget -q $S5CMD_URI -O s5cmd.tar.gz
-tar -xf s5cmd.tar.gz
-chmod +x s5cmd
-mv s5cmd /usr/bin
-s5cmd version
+# install aria2 a p2p downloader
+
+if [ "$arch" == "x86_64" ]; then
+  wget https://github.com/q3aql/aria2-static-builds/releases/download/v1.36.0/aria2-1.36.0-linux-gnu-64bit-build1.tar.bz2
+  tar jxvf aria2-1.36.0-linux-gnu-64bit-build1.tar.bz2
+  cd aria2-1.36.0-linux-gnu-64bit-build1/
+  make install
+else
+  wget https://github.com/q3aql/aria2-static-builds/releases/download/v1.36.0/aria2-1.36.0-linux-gnu-arm-rbpi-build1.tar.bz2
+  tar jxvf aria2-1.36.0-linux-gnu-arm-rbpi-build1.tar.bz2
+  cd aria2-1.36.0-linux-gnu-arm-rbpi-build1/
+  make install
+fi
 
 # Base specific setup starts here
 
@@ -77,13 +82,19 @@ STACK_NAME=${_STACK_NAME_}
 RESTORE_FROM_SNAPSHOT=${_RESTORE_FROM_SNAPSHOT_}
 FORMAT_DISK=${_FORMAT_DISK_}
 NETWORK_ID=${_NETWORK_ID_}
+NODE_CONFIG=${_NODE_CONFIG_}
 L1_EXECUTION_ENDPOINT=${_L1_EXECUTION_ENDPOINT_}
 L1_CONSENSUS_ENDPOINT=${_L1_CONSENSUS_ENDPOINT_}
+SNAPSHOT_URL=${_SNAPSHOT_URL_}
 
-echo "REGION=$REGION" >> /etc/environment
-echo "NETWORK_ID=$NETWORK_ID" >> /etc/environment
-echo "L1_EXECUTION_ENDPOINT=$L1_EXECUTION_ENDPOINT" >> /etc/environment
-echo "L1_CONSENSUS_ENDPOINT=$L1_CONSENSUS_ENDPOINT" >> /etc/environment
+{ 
+  echo "REGION=$REGION"
+  echo "NETWORK_ID=$NETWORK_ID"
+  echo "NODE_CONFIG=$NODE_CONFIG"
+  echo "L1_EXECUTION_ENDPOINT=$L1_EXECUTION_ENDPOINT"
+  echo "L1_CONSENSUS_ENDPOINT=$L1_CONSENSUS_ENDPOINT"
+  echo "SNAPSHOT_URL=$SNAPSHOT_URL"
+} >> /etc/environment
 
 GIT_URL=https://github.com/base-org/node.git
 SYNC_CHECKER_FILE_NAME=syncchecker-base.sh
@@ -147,16 +158,42 @@ echo "Configuring node"
 
 case $NETWORK_ID in
   "mainnet")
-    sed -i "s#OP_NODE_L1_ETH_RPC=https://1rpc.io/eth#OP_NODE_L1_ETH_RPC=$L1_EXECUTION_ENDPOINT#g" /home/bcuser/node/.env.mainnet
+    OP_CONFIG_FILE_PATH=/home/bcuser/node/.env.mainnet
+    ;;
+  "sepolia")
+    OP_CONFIG_FILE_PATH=/home/bcuser/node/.env.sepolia
+    ;;
+  *)
+    echo "Network id is not valid."
+    exit 1
+    ;;
+esac
+
+case $NODE_CONFIG in
+  "full")
+    echo "OP_GETH_GCMODE=full" >> $OP_CONFIG_FILE_PATH
+    ;;
+  "archive")
+     echo "OP_GETH_GCMODE=archive" >> $OP_CONFIG_FILE_PATH
+    ;;
+  *)
+    echo "Network id is not valid."
+    exit 1
+    ;;
+esac
+
+case $NETWORK_ID in
+  "mainnet")
+    sed -i "s#OP_NODE_L1_ETH_RPC=https://1rpc.io/eth#OP_NODE_L1_ETH_RPC=$L1_EXECUTION_ENDPOINT#g" $OP_CONFIG_FILE_PATH
     sed -i '/.env.mainnet/s/^#//g' /home/bcuser/node/docker-compose.yml
-    sed -i '/OP_NODE_L1_BEACON/s/^#//g' /home/bcuser/node/.env.mainnet
-    sed -i "s#OP_NODE_L1_BEACON=https://your.mainnet.beacon.node/endpoint-here#OP_NODE_L1_BEACON=$L1_CONSENSUS_ENDPOINT#g" /home/bcuser/node/.env.mainnet
+    sed -i '/OP_NODE_L1_BEACON/s/^#//g' $OP_CONFIG_FILE_PATH
+    sed -i "s#OP_NODE_L1_BEACON=https://your.mainnet.beacon.node/endpoint-here#OP_NODE_L1_BEACON=$L1_CONSENSUS_ENDPOINT#g" $OP_CONFIG_FILE_PATH
     ;;
   "sepolia")
-    sed -i "s#OP_NODE_L1_ETH_RPC=https://rpc.sepolia.org#OP_NODE_L1_ETH_RPC=$L1_EXECUTION_ENDPOINT#g" /home/bcuser/node/.env.sepolia
+    sed -i "s#OP_NODE_L1_ETH_RPC=https://rpc.sepolia.org#OP_NODE_L1_ETH_RPC=$L1_EXECUTION_ENDPOINT#g" $OP_CONFIG_FILE_PATH
     sed -i "/.env.sepolia/s/^#//g" /home/bcuser/node/docker-compose.yml
-    sed -i '/OP_NODE_L1_BEACON/s/^#//g' /home/bcuser/node/.env.sepolia
-    sed -i "s#OP_NODE_L1_BEACON=https://your.sepolia.beacon.node/endpoint-here#OP_NODE_L1_BEACON=$L1_CONSENSUS_ENDPOINT#g" /home/bcuser/node/.env.sepolia
+    sed -i '/OP_NODE_L1_BEACON/s/^#//g' $OP_CONFIG_FILE_PATH
+    sed -i "s#OP_NODE_L1_BEACON=https://your.sepolia.beacon.node/endpoint-here#OP_NODE_L1_BEACON=$L1_CONSENSUS_ENDPOINT#g" $OP_CONFIG_FILE_PATH
     ;;
   *)
     echo "Network id is not valid."
@@ -217,8 +254,8 @@ if [ "$RESTORE_FROM_SNAPSHOT" == "false" ]; then
   echo "sudo su bcuser && /usr/local/bin/docker-compose -f /home/bcuser/node/docker-compose.yml up -d" | at now +3 minutes
 else
   echo "Restoring data from snapshot"
-  chmod 766 /opt/restore-from-snapshot.sh
-  echo "/opt/restore-from-snapshot.sh" | at now +3 minutes
+  chmod 766 /opt/start-from-snapshot.sh
+  echo "/opt/start-from-snapshot.sh" | at now +3 minutes
 fi
 
 echo "All Done!!"
diff --git a/lib/base/lib/config/baseConfig.interface.ts b/lib/base/lib/config/baseConfig.interface.ts
@@ -1,7 +1,7 @@
 import * as configTypes from "../../../constructs/config.interface";
 
-export type BaseNetworkId = "mainnet" ;
-export type BaseNodeConfiguration = "full" ;
+export type BaseNetworkId = "mainnet" | "sepolia";
+export type BaseNodeConfiguration = "full" | "archive";
 
 export {AMBEthereumNodeNetworkId} from "../../../constructs/config.interface";
 
@@ -16,8 +16,10 @@ export interface BaseBaseConfig extends configTypes.BaseConfig {
 
 export interface BaseBaseNodeConfig extends configTypes.BaseNodeConfig {
     baseNetworkId: BaseNetworkId;
+    baseNodeConfiguration: BaseNodeConfiguration;
     dataVolume: BaseDataVolumeConfig;
     restoreFromSnapshot: boolean;
     l1ExecutionEndpoint: string;
     l1ConsensusEndpoint: string;
+    snapshotUrl: string;
 }
diff --git a/lib/base/lib/config/baseConfig.ts b/lib/base/lib/config/baseConfig.ts
@@ -27,9 +27,11 @@ export const baseNodeConfig: configTypes.BaseBaseNodeConfig = {
     instanceType: new ec2.InstanceType(process.env.BASE_INSTANCE_TYPE ? process.env.BASE_INSTANCE_TYPE : "m7g.2xlarge"),
     instanceCpuType: process.env.BASE_CPU_TYPE?.toLowerCase() == "x86_64" ? ec2.AmazonLinuxCpuType.X86_64 : ec2.AmazonLinuxCpuType.ARM_64,
     baseNetworkId: <configTypes.BaseNetworkId> process.env.BASE_NETWORK_ID || "mainnet",
+    baseNodeConfiguration: <configTypes.BaseNodeConfiguration> process.env.BASE_NODE_CONFIGURATION || "full",
     restoreFromSnapshot: process.env.BASE_RESTORE_FROM_SNAPSHOT?.toLowerCase() == "true" ? true : false,
     l1ExecutionEndpoint: process.env.BASE_L1_EXECUTION_ENDPOINT || constants.NoneValue,
     l1ConsensusEndpoint: process.env.BASE_L1_CONSENSUS_ENDPOINT || constants.NoneValue,
+    snapshotUrl: process.env.BASE_SNAPSHOT_URL || constants.NoneValue,
     dataVolume: {
         sizeGiB: process.env.BASE_DATA_VOL_SIZE ? parseInt(process.env.BASE_DATA_VOL_SIZE): 1000,
         type: parseDataVolumeType(process.env.BASE_DATA_VOL_TYPE?.toLowerCase() ? process.env.BASE_DATA_VOL_TYPE?.toLowerCase() : "gp3"),
diff --git a/lib/base/lib/single-node-stack.ts b/lib/base/lib/single-node-stack.ts
@@ -17,9 +17,11 @@ export interface BaseSingleNodeStackProps extends cdk.StackProps {
     instanceType: ec2.InstanceType;
     instanceCpuType: ec2.AmazonLinuxCpuType;
     baseNetworkId: configTypes.BaseNetworkId;
+    baseNodeConfiguration: configTypes.BaseNodeConfiguration;
     restoreFromSnapshot: boolean;
     l1ExecutionEndpoint: string,
     l1ConsensusEndpoint: string,
+    snapshotUrl: string,
     dataVolume: configTypes.BaseDataVolumeConfig;
 }
 
@@ -39,6 +41,7 @@ export class BaseSingleNodeStack extends cdk.Stack {
             instanceType,
             instanceCpuType,
             baseNetworkId,
+            baseNodeConfiguration,
             restoreFromSnapshot,
             l1ExecutionEndpoint,
             l1ConsensusEndpoint,
@@ -103,12 +106,14 @@ export class BaseSingleNodeStack extends cdk.Stack {
             _DATA_VOLUME_TYPE_: dataVolume.type,
             _DATA_VOLUME_SIZE_: dataVolumeSizeBytes.toString(),
             _NETWORK_ID_: baseNetworkId,
+            _NODE_CONFIG_: baseNodeConfiguration,
             _LIFECYCLE_HOOK_NAME_: constants.NoneValue,
             _AUTOSCALING_GROUP_NAME_: constants.NoneValue,
             _RESTORE_FROM_SNAPSHOT_: restoreFromSnapshot.toString(),
             _FORMAT_DISK_: "true",
             _L1_EXECUTION_ENDPOINT_: l1ExecutionEndpoint,
             _L1_CONSENSUS_ENDPOINT_: l1ConsensusEndpoint,
+            _SNAPSHOT_URL_: props.snapshotUrl,
         });
 
         node.instance.addUserData(modifiedInitNodeScript);
diff --git a/lib/base/sample-configs/.env-sample-rpc b/lib/base/sample-configs/.env-sample-rpc
@@ -8,6 +8,7 @@ AWS_REGION="us-east-1"
 
 ## Common configuration parameters ##
 BASE_NETWORK_ID="sepolia"              # All options:  "mainnet", "sepolia"
+BASE_NODE_CONFIGURATION="full"         # All options: "full", "archive"
 BASE_INSTANCE_TYPE="m7g.2xlarge"
 BASE_CPU_TYPE="ARM_64"                 # All options: "x86_64", "ARM_64". IMPORTANT: Make sure the CPU type matches the instance type used
 
@@ -20,6 +21,8 @@ BASE_RESTORE_FROM_SNAPSHOT="true"      # Download snapshot to speed up statup ti
 BASE_L1_EXECUTION_ENDPOINT="https://ethereum-sepolia-rpc.publicnode.com"        # Set your own URL to Ethereum L1 node: https://docs.base.org/tools/node-providers
 BASE_L1_CONSENSUS_ENDPOINT="https://ethereum-sepolia-beacon-api.publicnode.com"
 
+BASE_SNAPSHOT_URL="none"                # Optionally provide the URL to download snpashot: https://docs.base.org/tutorials/run-a-base-node/#snapshots
+
 # Example for Sepolia:
 #BASE_L1_EXECUTION_ENDPOINT=https://ethereum-sepolia-rpc.publicnode.com
 #BASE_L1_CONSENSUS_ENDPOINT=https://ethereum-sepolia-beacon-api.publicnode.com
