diff --git a/lib/ethereum/.env-sample b/lib/ethereum/.env-sample deleted file mode 100644 index f9971787..00000000 --- a/lib/ethereum/.env-sample +++ /dev/null @@ -1,29 +0,0 @@ -############################################################## -# Example configuration for Ethereum nodes runner app on AWS # -############################################################## - -# Set the AWS account is and region for your environment -AWS_ACCOUNT_ID="xxxxxxxxxxx" -AWS_REGION="us-east-2" - -# Common configuration parameters -ETH_CLIENT_COMBINATION="geth-lighthouse" # All options: "besu-teku" | "geth-lighthouse" | "erigon-lighthouse" | "erigon-prysm" | "nethermind-teku" - -# Sync node configuration -ETH_SYNC_INSTANCE_TYPE="m6g.2xlarge" -ETH_SYNC_CPU_TYPE="ARM_64" # IMPORTANT: Make sure the CPU type matches the instance type used -ETH_SYNC_DATA_VOL_SIZE="3072" # Minimum values in Gibibytes: nethermind-teku: 2048, geth-lighthouse: 2048, besu-teku: 2048, erigon-lighthouse: 3072 -ETH_SYNC_DATA_VOL_TYPE="gp3" # Other options: "io1" | "io2" | "gp3" | "instance-store" . IMPORTANT: Use "instance-store" option only with instance types that support that feature, like popular for node im4gn, d3, i3en, and i4i instance families -ETH_SYNC_DATA_VOL_IOPS="6000" # Max IOPS for EBS volumes (not applicable for "instance-store") -ETH_SYNC_DATA_VOL_THROUGHPUT="400" # Max throughput for EBS gp3 volumes (not applicable for "io1" | "io2" | "instance-store") - -# RPC nodes configuration -ETH_RPC_INSTANCE_TYPE="m7g.2xlarge" -ETH_RPC_CPU_TYPE="ARM_64" # IMPORTANT: Make sure the CPU type matches the instance type used -ETH_RPC_NUMBER_OF_NODES="2" # Total number of RPC nodes to be provisioned. Default: 2 -ETH_RPC_ALB_HEALTHCHECK_GRACE_PERIOD_MIN="10" # Time enough to initialize the instance and start downloading snpashots -ETH_RPC_HA_NODES_HEARTBEAT_DELAY_MIN="60" # Time sufficient enough for a node do download snapshot, from S3 bucket, start clients, and finish sycning the delta -ETH_RPC_DATA_VOL_SIZE="3072" # Minimum values in Gibibytes: nethermind-teku: 2048, geth-lighthouse: 2048, besu-teku: 2048, erigon-lighthouse: 3072 -ETH_RPC_DATA_VOL_TYPE="gp3" # Other options: "io1" | "io2" | "gp3" | "instance-store" . IMPORTANT: Use "instance-store" option only with instance types that support that feature, like popular for node im4gn, d3, i3en, and i4i instance families -ETH_RPC_DATA_VOL_IOPS="8000" # Max IOPS for EBS volumes (not applicable for "instance-store") -ETH_RPC_DATA_VOL_THROUGHPUT="700" # Max throughput for EBS gp3 volumes (not applicable for "io1" | "io2" | "instance-store") diff --git a/lib/ethereum/README.md b/lib/ethereum/README.md index 1657b618..35a6c2d7 100644 --- a/lib/ethereum/README.md +++ b/lib/ethereum/README.md @@ -59,7 +59,7 @@ This is the Well-Architected checklist for Ethereum nodes implementation of the ### Open AWS CloudShell -To begin, ensure you login to your AWS account with permissions to create and modify resources in IAM, EC2, EBS, VPC, S3, KMS, and Secrets Manager. +To begin, ensure you login to your AWS account with permissions to create and modify resources in IAM, EC2, EBS, VPC, and S3. From the AWS Management Console, open the [AWS CloudShell](https://docs.aws.amazon.com/cloudshell/latest/userguide/welcome.html), a web-based shell environment. If unfamiliar, review the [2-minute YouTube video](https://youtu.be/fz4rbjRaiQM) for an overview and check out [CloudShell with VPC environment](https://docs.aws.amazon.com/cloudshell/latest/userguide/creating-vpc-environment.html) that we'll use to test nodes API from internal IP address space. @@ -73,9 +73,7 @@ cd aws-blockchain-node-runners npm install ``` -> **NOTE:** *In this tutorial we will set all major configuration through environment variables, but you also can modify parameters in `config/config.ts`.* - -### Prepare to deploy nodes +### Prepare AWS account to deploy nodes 1. Make sure you are in the root directory of the cloned repository @@ -89,11 +87,11 @@ aws ec2 create-default-vpc > **NOTE:** *The default VPC must have at least two public subnets in different Availability Zones, and public subnet must set `Auto-assign public IPv4 address` to `YES`* -3. With the [Node Runners blueprints for Ethereum](https://github.com/aws-samples/aws-blockchain-node-runners/tree/main/lib/ethereum), you can deploy both single Ethereum nodes and multi-node high-availability configurations on AWS. Furthermore, Node Runners is designed to support client diversity, with configurations available for a variety of client combinations for the Execution Layer (EL) and Consensus Layer (CL). +### Configure your setup -Configure your setup. +#### Execution and Consensus Layer Client Options -### Execution and Consensus Layer Client Options +With the [Node Runners blueprints for Ethereum](https://github.com/aws-samples/aws-blockchain-node-runners/tree/main/lib/ethereum), you can deploy both single Ethereum nodes and multi-node high-availability configurations on AWS. Furthermore, Node Runners is designed to support client diversity, with configurations available for a variety of client combinations for the Execution Layer (EL) and Consensus Layer (CL).
@@ -135,17 +133,17 @@ nano .env
-Erigon Prysm +Reth Lighthouse
-**Configure your Node Runners Ethereum - Erigon Prysm** +**Configure your Node Runners Ethereum - Reth Lighthouse** -To specify the Ethereum client combination you wish to deploy, create your own copy of `.env` file and edit it using your preferred text editor. The contents of your file for a Erigon / Prysm node deployment is as follows, which uses a sample config from the repository: +To specify the Ethereum client combination you wish to deploy, create your own copy of `.env` file and edit it using your preferred text editor. The contents of your file for a Reth / Lighthouse node deployment is as follows, which uses a sample config from the repository: ```bash # Make sure you are in aws-blockchain-node-runners/lib/ethereum cd lib/ethereum pwd -cp ./sample-configs/.env-erigon-prysm .env +cp ./sample-configs/.env-erigon-lighthouse .env nano .env ``` > **NOTE:** *You can find more examples inside the `sample-configs` directory, which illustrate other Ethereum client combinations.* @@ -201,9 +199,32 @@ pwd npx cdk deploy eth-common ``` -### Option 1: Single RPC Node +### [OPTIONAL] (required only when ETH_SNAPSHOT_TYPE="s3") Deploy Sync Node + +Sync node will sync with the network and periodically create data snapshots on S3 to speed up RPC nodes setup when `ETH_SNAPSHOT_TYPE="s3"`. It has no effect if `ETH_SNAPSHOT_TYPE="none"`. + +1. Deploy `eth-sync-node` stack + +```bash +pwd +# Make sure you are in aws-blockchain-node-runners/lib/ethereum +npx cdk deploy eth-sync-node --json --outputs-file sync-node-deploy.json +``` + +2. After starting the node you need to wait for the inital syncronization process to finish. It may take from half a day to about 6-10 days depending on the client combination and the state of the network. You can use Amazon CloudWatch to track the progress. There is a script that publishes CloudWatch metrics every 5 minutes, where you can watch `sync distance` for consensus client and `blocks behind` for execution client. When the node is fully synced those two metrics shold show 0. To see them: + + - Navigate to [CloudWatch service](https://console.aws.amazon.com/cloudwatch/) (make sure you are in the region you have specified for `AWS_REGION`) + - Open `Dashboards` and select `eth-sync-node-` from the list of dashboards. + +Once synchronization process is over, the script will automatically stop both clients and copy all the contents of the `/data` directory to your snapshot S3 bucket. That may take from 30 minutes to about 2 hours. During the process on the dashboard you will see lower CPU and RAM utilization but high data disc throughput and outbound network traffic. The script will automatically start the clients after the process is done. + +> **Note:** *The snapshot backup process will automatically run ever day at midnight time of the time zone were the sync node runs. To change the schedule, modify `crontab` of the root user on the node's EC2 instance.* + +### Deploy Standalone RPC Node + +> **NOTE:** *If `ETH_SNAPSHOT_TYPE="s3"` make sure you [deployed the Sync Node first](#optional-required-only-when-eth_snapshot_types3-deploy-sync-node).* -1. Deploy Single RPC Node +1. Deploy `eth-single-node` stack ```bash pwd @@ -212,12 +233,12 @@ npx cdk deploy eth-single-node --json --outputs-file single-node-deploy.json ``` > **NOTE:** *The default VPC must have at least two public subnets in different Availability Zones, and public subnet must set `Auto-assign public IPv4 address` to `YES`* -2. After starting the node you need to wait for the inital syncronization process to finish. It may take from half a day to about 6-10 days depending on the client combination and the state of the network. You can use Amazon CloudWatch to track the progress. There is a script that publishes CloudWatch metrics every 5 minutes, where you can watch `sync distance` for consensus client and `blocks behind` for execution client. When the node is fully synced those two metrics shold show 0. To see them: +2. If you haven't used `ETH_SNAPSHOT_TYPE="s3"` with Sync Node, then your node will start syncing by itself. In that case, after starting the node you need to wait for the inital syncronization process to finish. It may take from half a day to about 6-10 days depending on the client combination and the state of the network. You can use Amazon CloudWatch to track the progress. There is a script that publishes CloudWatch metrics every 5 minutes, where you can watch `sync distance` for consensus client and `blocks behind` for execution client. When the node is fully synced those two metrics shold show 0. To see them: - Navigate to [CloudWatch service](https://console.aws.amazon.com/cloudwatch/) (make sure you are in the region you have specified for `AWS_REGION`) - - Open `Dashboards` and select `eth-sync-node-` from the list of dashboards. + - Open `Dashboards` and select `eth-single-node-` from the list of dashboards. -4. Once the initial synchronization is done, you should be able to access the RPC API of that node from within the same VPC. The RPC port is not exposed to the Internet. Turn the following query against the private IP of the single RPC node you deployed: +3. Once the initial synchronization is done, you should be able to access the RPC API of that node from within the same VPC. The RPC port is not exposed to the Internet. Turn the following query against the private IP of the single RPC node you deployed: ```bash INSTANCE_ID=$(cat single-node-deploy.json | jq -r '..|.node-instance-id? | select(. != null)') @@ -240,27 +261,17 @@ The result should be like this (the actual balance might change): {"jsonrpc":"2.0","id":1,"result":"0xe791d050f91d9949d344d"} ``` -### Option 2: Highly Available RPC Nodes - -1. Deploy Sync Node - -```bash -pwd -# Make sure you are in aws-blockchain-node-runners/lib/ethereum -npx cdk deploy eth-sync-node --json --outputs-file sync-node-deploy.json -``` -**NOTE:** *The default VPC must have at least two public subnets in different Availability Zones, and public subnet must set `Auto-assign public IPv4 address` to `YES`* +### Deploy Highly Available RPC Nodes -2. After starting the node you need to wait for the inital syncronization process to finish. It may take from half a day to about 6-10 days depending on the client combination and the state of the network. You can use Amazon CloudWatch to track the progress. There is a script that publishes CloudWatch metrics every 5 minutes, where you can watch `sync distance` for consensus client and `blocks behind` for execution client. When the node is fully synced those two metrics shold show 0. To see them: +> **NOTE:** *If `ETH_SNAPSHOT_TYPE="s3"` make sure you [deployed the Sync Node first](#optional-required-only-when-eth_snapshot_types3-deploy-sync-node).* - - Navigate to [CloudWatch service](https://console.aws.amazon.com/cloudwatch/) (make sure you are in the region you have specified for `AWS_REGION`) - - Open `Dashboards` and select `eth-sync-node-` from the list of dashboards. +> **NOTE:** *The default VPC must have at least two public subnets in different Availability Zones, and public subnet must set `Auto-assign public IPv4 address` to `YES`* -Once synchronization process is over, the script will automatically stop both clients and copy all the contents of the `/data` directory to your snapshot S3 bucket. That may take from 30 minutes to about 2 hours. During the process on the dashboard you will see lower CPU and RAM utilization but high data disc throughput and outbound network traffic. The script will automatically start the clients after the process is done. +1. Deploy Sync Node -> **Note:** *The snapshot backup process will automatically run ever day at midnight time of the time zone were the sync node runs. To change the schedule, modify `crontab` of the root user on the node's EC2 instance.* +Use instructions from earlier: [Deploy Sync Node](#optional-required-only-when-eth_snapshot_types3-deploy-sync-node) -3. Configure and deploy 2 RPC Nodes +2. Deploy `eth-rpc-nodes` stack ```bash pwd @@ -268,7 +279,7 @@ pwd npx cdk deploy eth-rpc-nodes --json --outputs-file rpc-node-deploy.json ``` -4. Give the new RPC nodes about 30 minutes (up to 2 hours for Erigon) to initialize and then run the following query against the load balancer behind the RPC node created +3. Give the new RPC nodes about 30 minutes (up to 2 hours for Erigon) to initialize and then run the following query against the load balancer in front of your nods: ```bash export ETH_RPC_ABL_URL=$(cat rpc-node-deploy.json | jq -r '..|.alburl? | select(. != null)') @@ -288,7 +299,7 @@ The result should be like this (the actual balance might change): {"jsonrpc":"2.0","id":1,"result":"0xe791d050f91d9949d344d"} ``` - If the nodes are still starting and catching up with the chain, you will see the following repsonse: + If the nodes are still starting and catching up with the chain, you will see the following response: ```HTML @@ -298,7 +309,7 @@ The result should be like this (the actual balance might change): ``` -> **NOTE:** By default and for security reasons the load balancer is available only from within the default VPC in the region where it is deployed. It is not available from the Internet and is not open for external connections. Before opening it up please make sure you protect your RPC APIs. +> **NOTE:** *By default and for security reasons the load balancer is available only from within the default VPC in the region where it is deployed. It is not available from the Internet and is not open for external connections. Before opening it up please make sure you protect your RPC APIs.* ### Clearing up and undeploying everything diff --git a/lib/ethereum/app.ts b/lib/ethereum/app.ts index 1197d359..f144932f 100644 --- a/lib/ethereum/app.ts +++ b/lib/ethereum/app.ts @@ -3,19 +3,20 @@ import 'dotenv/config' import "source-map-support/register"; import * as cdk from "aws-cdk-lib"; import * as nag from "cdk-nag"; -import * as config from "./lib/config/ethConfig"; -import { EthNodeRole } from "./lib/config/ethConfig.interface"; +import * as config from "./lib/config/node-config"; +import { EthNodeRole } from "./lib/config/node-config.interface"; import { EthSingleNodeStack } from "./lib/single-node-stack"; import { EthCommonStack } from "./lib/common-stack"; import { EthRpcNodesStack } from "./lib/rpc-nodes-stack"; const app = new cdk.App(); -cdk.Tags.of(app).add("Project", "Ethereum"); +cdk.Tags.of(app).add("Project", "AWSEthereum"); new EthCommonStack(app, "eth-common", { - env: { account: config.baseConfig.accountId, region: config.baseConfig.region }, stackName: `eth-nodes-common`, + env: { account: config.baseConfig.accountId, region: config.baseConfig.region }, + snapshotType: config.baseConfig.snapshotType, }); new EthSingleNodeStack(app, "eth-sync-node", { @@ -23,10 +24,15 @@ new EthSingleNodeStack(app, "eth-sync-node", { env: { account: config.baseConfig.accountId, region: config.baseConfig.region }, ethClientCombination: config.baseConfig.clientCombination, + network: config.baseConfig.network, + snapshotType: config.baseConfig.snapshotType, + consensusSnapshotURL: config.baseConfig.consensusSnapshotURL, + executionSnapshotURL: config.baseConfig.executionSnapshotURL, + consensusCheckpointSyncURL: config.baseConfig.consensusCheckpointSyncURL, nodeRole: "sync-node", instanceType: config.syncNodeConfig.instanceType, instanceCpuType: config.syncNodeConfig.instanceCpuType, - dataVolumes: config.syncNodeConfig.dataVolumes, + dataVolume: config.syncNodeConfig.dataVolumes[0], }); new EthSingleNodeStack(app, "eth-single-node", { @@ -34,10 +40,15 @@ new EthSingleNodeStack(app, "eth-single-node", { env: { account: config.baseConfig.accountId, region: config.baseConfig.region }, ethClientCombination: config.baseConfig.clientCombination, + network: config.baseConfig.network, + snapshotType: config.baseConfig.snapshotType, + consensusSnapshotURL: config.baseConfig.consensusSnapshotURL, + executionSnapshotURL: config.baseConfig.executionSnapshotURL, + consensusCheckpointSyncURL: config.baseConfig.consensusCheckpointSyncURL, nodeRole: "single-node", - instanceType: config.syncNodeConfig.instanceType, - instanceCpuType: config.syncNodeConfig.instanceCpuType, - dataVolumes: config.syncNodeConfig.dataVolumes, + instanceType: config.rpcNodeConfig.instanceType, + instanceCpuType: config.rpcNodeConfig.instanceCpuType, + dataVolume: config.rpcNodeConfig.dataVolumes[0], }); new EthRpcNodesStack(app, "eth-rpc-nodes", { @@ -45,13 +56,18 @@ new EthRpcNodesStack(app, "eth-rpc-nodes", { env: { account: config.baseConfig.accountId, region: config.baseConfig.region }, ethClientCombination: config.baseConfig.clientCombination, + network: config.baseConfig.network, + snapshotType: config.baseConfig.snapshotType, + consensusSnapshotURL: config.baseConfig.consensusSnapshotURL, + executionSnapshotURL: config.baseConfig.executionSnapshotURL, + consensusCheckpointSyncURL: config.baseConfig.consensusCheckpointSyncURL, nodeRole: "rpc-node", instanceType: config.rpcNodeConfig.instanceType, instanceCpuType: config.rpcNodeConfig.instanceCpuType, numberOfNodes: config.rpcNodeConfig.numberOfNodes, albHealthCheckGracePeriodMin: config.rpcNodeConfig.albHealthCheckGracePeriodMin, heartBeatDelayMin: config.rpcNodeConfig.heartBeatDelayMin, - dataVolumes: config.syncNodeConfig.dataVolumes, + dataVolume: config.syncNodeConfig.dataVolumes[0], }); diff --git a/lib/ethereum/lib/assets/copy-data-to-s3.sh b/lib/ethereum/lib/assets/copy-data-to-s3.sh deleted file mode 100644 index 63cef98a..00000000 --- a/lib/ethereum/lib/assets/copy-data-to-s3.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set +e -source /etc/environment - -/usr/local/bin/docker-compose -f /home/ethereum/docker-compose.yml down -echo "Sync started at " $(date) -s5cmd --log error sync /data $SNAPSHOT_S3_PATH/ -echo "Sync finished at " $(date) -sudo touch /data/snapshotted -sudo su ethereum -/usr/local/bin/docker-compose -f /home/ethereum/docker-compose.yml up -d diff --git a/lib/ethereum/lib/assets/instance/cfn-hup/cfn-auto-reloader.conf b/lib/ethereum/lib/assets/instance/cfn-hup/cfn-auto-reloader.conf new file mode 100644 index 00000000..3cd32a0a --- /dev/null +++ b/lib/ethereum/lib/assets/instance/cfn-hup/cfn-auto-reloader.conf @@ -0,0 +1,4 @@ +[cfn-auto-reloader-hook] +triggers=post.update +path=Resources.WebServerHost.Metadata.AWS::CloudFormation::Init +action=/opt/aws/bin/cfn-init -v --stack __AWS_STACK_NAME__ --resource WebServerHost --region __AWS_REGION__ diff --git a/lib/ethereum/lib/assets/instance/cfn-hup/cfn-hup.conf b/lib/ethereum/lib/assets/instance/cfn-hup/cfn-hup.conf new file mode 100644 index 00000000..2163b37a --- /dev/null +++ b/lib/ethereum/lib/assets/instance/cfn-hup/cfn-hup.conf @@ -0,0 +1,5 @@ +[main] +stack=__AWS_STACK_ID__ +region=__AWS_REGION__ +# The interval used to check for changes to the resource metadata in minutes. Default is 15 +interval=2 diff --git a/lib/ethereum/lib/assets/instance/cfn-hup/cfn-hup.service b/lib/ethereum/lib/assets/instance/cfn-hup/cfn-hup.service new file mode 100644 index 00000000..2660ea46 --- /dev/null +++ b/lib/ethereum/lib/assets/instance/cfn-hup/cfn-hup.service @@ -0,0 +1,8 @@ +[Unit] +Description=cfn-hup daemon +[Service] +Type=simple +ExecStart=/usr/local/bin/cfn-hup +Restart=always +[Install] +WantedBy=multi-user.target diff --git a/lib/ethereum/lib/assets/instance/cfn-hup/setup.sh b/lib/ethereum/lib/assets/instance/cfn-hup/setup.sh new file mode 100755 index 00000000..418811e4 --- /dev/null +++ b/lib/ethereum/lib/assets/instance/cfn-hup/setup.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +if [ -n "$1" ]; then + export STACK_ID=$1 +else + echo "Error: No Stack ID is provided" + echo "Usage: instance/cfn-hup/setup.sh " + exit 1 +fi + +if [ -n "$2" ]; then + export AWS_REGION=$2 +else + echo "Error: No AWS Region is provided" + echo "Usage: instance/cfn-hup/setup.sh " + exit 1 +fi + + echo "Install CloudFormation helper scripts" + mkdir -p /opt/aws/ + pip3 install --break-system-packages https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz + ln -s /usr/local/init/ubuntu/cfn-hup /etc/init.d/cfn-hup + + echo "Configuring CloudFormation helper scripts" + mkdir -p /etc/cfn/ + mv /opt/instance/cfn-hup/cfn-hup.conf /etc/cfn/cfn-hup.conf + sed -i "s;__AWS_STACK_ID__;\"$STACK_ID\";g" /etc/cfn/cfn-hup.conf + sed -i "s;__AWS_REGION__;\"$AWS_REGION\";g" /etc/cfn/cfn-hup.conf + + mkdir -p /etc/cfn/hooks.d/system + mv /opt/instance/cfn-hup/cfn-auto-reloader.conf /etc/cfn/hooks.d/cfn-auto-reloader.conf + sed -i "s;__AWS_STACK_NAME__;\"$STACK_NAME\";g" /etc/cfn/hooks.d/cfn-auto-reloader.conf + sed -i "s;__AWS_REGION__;\"$AWS_REGION\";g" /etc/cfn/hooks.d/cfn-auto-reloader.conf + + echo "Starting CloudFormation helper scripts as a service" + mv /opt/instance/cfn-hup/cfn-hup.service /etc/systemd/system/cfn-hup.service + + systemctl daemon-reload + systemctl enable --now cfn-hup + systemctl start cfn-hup.service diff --git a/lib/ethereum/lib/assets/copy-data-from-s3.sh b/lib/ethereum/lib/assets/instance/storage/copy-data-from-s3.sh similarity index 85% rename from lib/ethereum/lib/assets/copy-data-from-s3.sh rename to lib/ethereum/lib/assets/instance/storage/copy-data-from-s3.sh index aa70fd09..b3d0aac2 100644 --- a/lib/ethereum/lib/assets/copy-data-from-s3.sh +++ b/lib/ethereum/lib/assets/instance/storage/copy-data-from-s3.sh @@ -1,6 +1,6 @@ #!/bin/bash -source /etc/environment +source /etc/cdk_environment TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/instance-id) @@ -8,10 +8,10 @@ echo "Sync started at " $(date) SECONDS=0 s5cmd --log error cp --exclude 'lost+found' $SNAPSHOT_S3_PATH/data/* /data && \ -chown -R ethereum:ethereum /data && \ +chown -R bcuser:bcuser /data && \ echo "Sync finished at " $(date) && \ echo "$(($SECONDS / 60)) minutes and $(($SECONDS % 60)) seconds elapsed." && \ -su ethereum && \ -/usr/local/bin/docker-compose -f /home/ethereum/docker-compose.yml up -d && \ +su bcuser && \ +docker compose -f /home/bcuser/docker-compose.yml up -d && \ aws autoscaling complete-lifecycle-action --lifecycle-action-result CONTINUE --instance-id $INSTANCE_ID --lifecycle-hook-name "$LIFECYCLE_HOOK_NAME" --auto-scaling-group-name "$AUTOSCALING_GROUP_NAME" --region $REGION || \ aws autoscaling complete-lifecycle-action --lifecycle-action-result ABANDON --instance-id $INSTANCE_ID --lifecycle-hook-name "$LIFECYCLE_HOOK_NAME" --auto-scaling-group-name "$AUTOSCALING_GROUP_NAME" --region $REGION diff --git a/lib/ethereum/lib/assets/instance/storage/copy-data-to-s3.sh b/lib/ethereum/lib/assets/instance/storage/copy-data-to-s3.sh new file mode 100644 index 00000000..6ecbf81f --- /dev/null +++ b/lib/ethereum/lib/assets/instance/storage/copy-data-to-s3.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set +e +source /etc/cdk_environment + +/usr/local/bin/docker-compose -f /home/bcuser/docker-compose.yml down +echo "Sync started at " $(date) +s5cmd --log error sync /data $SNAPSHOT_S3_PATH/ +echo "Sync finished at " $(date) +sudo touch /data/snapshotted +sudo su bcuser +docker compose -f /home/bcuser/docker-compose.yml up -d diff --git a/lib/ethereum/lib/assets/instance/storage/setup.sh b/lib/ethereum/lib/assets/instance/storage/setup.sh new file mode 100755 index 00000000..ca5e6b81 --- /dev/null +++ b/lib/ethereum/lib/assets/instance/storage/setup.sh @@ -0,0 +1,130 @@ +#!/bin/bash + +make_fs () { + # If file system = to ext4 use mkfs.ext4, if xfs use mkfs.xfs + if [ -z "$1" ]; then + echo "Error: No file system type provided." + echo "Usage: make_fs " + exit 1 + fi + + if [ -z "$2" ]; then + echo "Error: No target volume ID provided." + echo "Usage: make_fs " + exit 1 + fi + + local file_system=$1 + local volume_id=$2 + if [ "$file_system" == "ext4" ]; then + mkfs -t ext4 "$volume_id" + return "$?" + else + mkfs.xfs -f "$volume_id" + return "$?" + fi +} + +# We need an nvme disk that is not mounted and not partitioned +get_all_empty_nvme_disks () { + local all_not_mounted_nvme_disks + local all_mounted_nvme_partitions + local unmounted_nvme_disks=() + local sorted_unmounted_nvme_disks + + #The disk will only be mounted when the nvme disk is larger than 100GB to avoid storing blockchain node data directly on the root EBS disk (which is 46GB by default) + all_not_mounted_nvme_disks=$(lsblk -lnb | awk '{if ($7 == "" && $4 > 100000000) {print $1}}' | grep nvme) + all_mounted_nvme_partitions=$(mount | awk '{print $1}' | grep /dev/nvme) + for disk in ${all_not_mounted_nvme_disks[*]}; do + if [[ ! "${all_mounted_nvme_partitions[*]}" =~ $disk ]]; then + unmounted_nvme_disks+=("$disk") + fi + done + # Sort the array + sorted_unmounted_nvme_disks=($(printf '%s\n' "${unmounted_nvme_disks[*]}" | sort)) + echo "${sorted_unmounted_nvme_disks[*]}" +} + +get_next_empty_nvme_disk () { + local sorted_unmounted_nvme_disks + sorted_unmounted_nvme_disks=($(get_all_empty_nvme_disks)) + # Return the first unmounted nvme disk + echo "/dev/${sorted_unmounted_nvme_disks[0]}" +} + +# Add input as command line parameters for name of the directory to mount +if [ -n "$1" ]; then + DIR_NAME=$1 +else + echo "Error: No data file system mount path is provided." + echo "Usage: instance/storage/setup.sh " + echo "Default file system type is ext4" + echo "If you skip , script will try to use the first unformatted volume ID." + echo "Usage example: instance/storage/setup.sh /data ext4 300000000000000" + exit 1 +fi + +# Case input for $2 between ext4 and xfs, use ext4 as default +case $2 in + ext4) + echo "File system set to ext4" + FILE_SYSTEM="ext4" + FS_CONFIG="defaults" + ;; + xfs) + echo "File system set to xfs" + FILE_SYSTEM="xfs" + FS_CONFIG="noatime,nodiratime,nodiscard" # See more: https://cdrdv2-public.intel.com/686417/rocksdb-benchmark-tuning-guide-on-xeon.pdf + ;; + *) + echo "File system set to ext4" + FILE_SYSTEM="ext4" + FS_CONFIG="defaults" + ;; +esac + +if [ -n "$3" ]; then + VOLUME_SIZE=$3 +else + echo "The size of volume for $DIR_NAME is not specified. Will try to guess volume ID." +fi + + echo "Checking if $DIR_NAME is mounted, and dont do anything if it is" + if [ $(df --output=target | grep -c "$DIR_NAME") -lt 1 ]; then + + if [ -n "$VOLUME_SIZE" ]; then + VOLUME_ID=/dev/$(lsblk -lnb | awk -v VOLUME_SIZE_BYTES="$VOLUME_SIZE" '{if ($4== VOLUME_SIZE_BYTES) {print $1}}') + echo "Data volume size defined, use respective volume id: $VOLUME_ID" + else + VOLUME_ID=$(get_next_empty_nvme_disk) + echo "Data volume size undefined, trying volume id: $VOLUME_ID" + fi + + make_fs $FILE_SYSTEM "$VOLUME_ID" + + sleep 10 + VOLUME_UUID=$(lsblk -fn -o UUID "$VOLUME_ID") + VOLUME_FSTAB_CONF="UUID=$VOLUME_UUID $DIR_NAME $FILE_SYSTEM $FS_CONFIG 0 2" + echo "VOLUME_ID=$VOLUME_ID" + echo "VOLUME_UUID=$VOLUME_UUID" + echo "VOLUME_FSTAB_CONF=$VOLUME_FSTAB_CONF" + + # Check if data disc is already in fstab and replace the line if it is with the new disc UUID + echo "Checking fstab for volume $DIR_NAME" + if [ $(grep -c "$DIR_NAME" /etc/fstab) -gt 0 ]; then + SED_REPLACEMENT_STRING="$(grep -n "$DIR_NAME" /etc/fstab | cut -d: -f1)s#.*#$VOLUME_FSTAB_CONF#" + # if file exists, delete it + if [ -f /etc/fstab.bak ]; then + rm /etc/fstab.bak + fi + cp /etc/fstab /etc/fstab.bak + sed -i "$SED_REPLACEMENT_STRING" /etc/fstab + else + echo "$VOLUME_FSTAB_CONF" | tee -a /etc/fstab + fi + + mount -a + chown -R bcuser:bcuser "$DIR_NAME" + else + echo "$DIR_NAME volume is mounted, nothing changed" + fi diff --git a/lib/ethereum/lib/assets/docker-compose/docker-compose-besu-teku.yml b/lib/ethereum/lib/assets/node/docker-compose-besu-teku.yml similarity index 89% rename from lib/ethereum/lib/assets/docker-compose/docker-compose-besu-teku.yml rename to lib/ethereum/lib/assets/node/docker-compose-besu-teku.yml index 01b21770..e21f57d9 100644 --- a/lib/ethereum/lib/assets/docker-compose/docker-compose-besu-teku.yml +++ b/lib/ethereum/lib/assets/node/docker-compose-besu-teku.yml @@ -4,12 +4,12 @@ services: besu_node: environment: - "JAVA_OPTS=-Xmx8g" - image: hyperledger/besu:24.3.0 + image: hyperledger/besu:25.1.0 container_name: execution restart: always command: [ - "--network=mainnet", + "--network=__ETH_NETWORK__", "--data-path=/var/lib/besu/data", "--sync-mode=X_SNAP", "--data-storage-format=BONSAI", @@ -22,7 +22,7 @@ services: "--engine-rpc-enabled=true", "--Xplugin-rocksdb-high-spec-enabled", ] - user: ethereum:ethereum + user: bcuser:bcuser volumes: - /secrets:/var/lib/besu/secrets - /data/execution/data:/var/lib/besu/data @@ -43,14 +43,14 @@ services: environment: - "JAVA_OPTS=-Xmx4g" - "TEKU_OPTS=-XX:-HeapDumpOnOutOfMemoryError" - image: consensys/teku:24.3.0-jdk17 + image: consensys/teku:25.1.0-jdk21 container_name: consensus restart: always command: [ "--data-base-path=/var/lib/teku/data", "--ee-endpoint=http://besu_node:8551", - "--initial-state=https://beaconstate.info/eth/v2/debug/beacon/states/finalized", + "--checkpoint-sync-url=__ETH_CONSENSUS_CHECKPOINT_SYNC_URL__", "--ee-jwt-secret-file=/var/lib/teku/secrets/jwtsecret", "--p2p-port=9000", "--metrics-enabled=true", @@ -58,7 +58,7 @@ services: ] depends_on: - besu_node - user: ethereum:ethereum + user: bcuser:bcuser volumes: - /secrets:/var/lib/teku/secrets - /data/consensus/data:/var/lib/teku/data diff --git a/lib/ethereum/lib/assets/docker-compose/docker-compose-erigon-lighthouse.yml b/lib/ethereum/lib/assets/node/docker-compose-erigon-lighthouse.yml similarity index 70% rename from lib/ethereum/lib/assets/docker-compose/docker-compose-erigon-lighthouse.yml rename to lib/ethereum/lib/assets/node/docker-compose-erigon-lighthouse.yml index c00e3b5f..03d4b530 100644 --- a/lib/ethereum/lib/assets/docker-compose/docker-compose-erigon-lighthouse.yml +++ b/lib/ethereum/lib/assets/node/docker-compose-erigon-lighthouse.yml @@ -2,12 +2,12 @@ version: "3" services: erigon_node: - image: thorax/erigon:2.59.0-__IMAGE_ARCH__ + image: erigontech/erigon:v2.61.0 container_name: execution restart: always command: [ - "--chain=mainnet", + "--chain=__ETH_NETWORK__", "--torrent.download.rate=1gb", "--datadir=/var/lib/erigon/data", "--authrpc.addr=0.0.0.0", @@ -33,27 +33,28 @@ services: read_only: true ports: # Map the p2p port(30303), RPC HTTP port(8545), and engine port (8551) - - "6060:6060" - - "8545:8545" - - "8546:8546" - - "8551:8551" - - "9093:9093" - - "9094:9094" - - "30303:30303/tcp" - - "30303:30303/udp" + - "9090:9090" # TCP gRPC Private Execution + - "8545:8545" # TCP HTTP & WebSockets & GraphQL Private Execution + - "8551:8551" # TCP Engine API (JWT auth) Private Execution + - "9093:9093" # Execution + - "9094:9094" # Execution + - "42069:42069/tcp" # TCP & UDP Snap sync Public (Bittorrent) + - "42069:42069/udp" # TCP & UDP Snap sync Public (Bittorrent) + - "30303:30303/tcp" # eth/68 peering Public Execution + - "30303:30303/udp" # eth/68 peering Public Execution lighthouse_node: - image: sigp/lighthouse:v5.1.2 + image: sigp/lighthouse:v6.0.1 container_name: consensus restart: always command: [ "lighthouse", "beacon", - "--network=mainnet", + "--network=__ETH_NETWORK__", "--execution-endpoint=http://erigon_node:8551", "--execution-jwt=/var/lib/lighthouse/secrets/jwtsecret", - "--checkpoint-sync-url=https://sync-mainnet.beaconcha.in", + "--checkpoint-sync-url=__ETH_CONSENSUS_CHECKPOINT_SYNC_URL__", "--http", "--http-port=5052", "--http-address=0.0.0.0", diff --git a/lib/ethereum/lib/assets/docker-compose/docker-compose-erigon-prysm.yml b/lib/ethereum/lib/assets/node/docker-compose-erigon-prysm.yml similarity index 70% rename from lib/ethereum/lib/assets/docker-compose/docker-compose-erigon-prysm.yml rename to lib/ethereum/lib/assets/node/docker-compose-erigon-prysm.yml index a93e9667..b3ff8855 100644 --- a/lib/ethereum/lib/assets/docker-compose/docker-compose-erigon-prysm.yml +++ b/lib/ethereum/lib/assets/node/docker-compose-erigon-prysm.yml @@ -2,12 +2,12 @@ version: "3" services: erigon_node: - image: thorax/erigon:2.59.0-__IMAGE_ARCH__ + image: erigontech/erigon:v2.61.0 container_name: execution restart: always command: [ - "--chain=mainnet", + "--chain=__ETH_NETWORK__", "--torrent.download.rate=1gb", "--datadir=/var/lib/erigon/data", "--authrpc.addr=0.0.0.0", @@ -33,25 +33,25 @@ services: read_only: true ports: # Map the p2p port(30303), RPC HTTP port(8545), and engine port (8551) - - "6060:6060" - - "8545:8545" - - "8546:8546" - - "8551:8551" - - "9093:9093" - - "9094:9094" - - "30303:30303/tcp" - - "30303:30303/udp" + - "9090:9090" # TCP gRPC Private Execution + - "8545:8545" # TCP HTTP & WebSockets & GraphQL Private Execution + - "8551:8551" # TCP Engine API (JWT auth) Private Execution + - "9093:9093" # Execution + - "9094:9094" # Execution + - "42069:42069/tcp" # TCP & UDP Snap sync Public (Bittorrent) + - "42069:42069/udp" # TCP & UDP Snap sync Public (Bittorrent) + - "30303:30303/tcp" # eth/68 peering Public Execution + - "30303:30303/udp" # eth/68 peering Public Execution prysm_node: - image: rocketpool/prysm:v5.0.1 + image: rocketpool/prysm:v5.2.0 container_name: consensus restart: always command: [ "/app/cmd/beacon-chain/beacon-chain", - "--mainnet", - "--checkpoint-sync-url=https://beaconstate.info", - "--genesis-beacon-api-url=https://beaconstate.info", + "--__ETH_NETWORK__", + "--checkpoint-sync-url=__ETH_CONSENSUS_CHECKPOINT_SYNC_URL__", "--execution-endpoint=http://erigon_node:8551", "--jwt-secret=/secrets/jwtsecret", "--datadir=/data", diff --git a/lib/ethereum/lib/assets/docker-compose/docker-compose-geth-lighthouse.yml b/lib/ethereum/lib/assets/node/docker-compose-geth-lighthouse.yml similarity index 87% rename from lib/ethereum/lib/assets/docker-compose/docker-compose-geth-lighthouse.yml rename to lib/ethereum/lib/assets/node/docker-compose-geth-lighthouse.yml index 5a5f713c..5bb9792f 100644 --- a/lib/ethereum/lib/assets/docker-compose/docker-compose-geth-lighthouse.yml +++ b/lib/ethereum/lib/assets/node/docker-compose-geth-lighthouse.yml @@ -2,12 +2,12 @@ version: "3" services: geth_node: - image: ethereum/client-go:v1.13.14 + image: ethereum/client-go:v1.14.12 container_name: execution restart: always command: [ - "--mainnet", + "--__ETH_NETWORK__", "--metrics", "--pprof", "--authrpc.addr=0.0.0.0", @@ -17,6 +17,8 @@ services: "--http", "--http.api=admin,eth,web3,txpool,net,debug,engine", "--http.addr=0.0.0.0", + "--ws", + "--db.engine=pebble", "--datadir=/var/lib/geth/data", "--maxpeers=20", "--cache=${CACHE_MB:-60000}", @@ -38,17 +40,17 @@ services: - "30303:30303/udp" lighthouse_node: - image: sigp/lighthouse:v5.1.2 + image: sigp/lighthouse:v6.0.1 container_name: consensus restart: always command: [ "lighthouse", "beacon", - "--network=mainnet", + "--network=__ETH_NETWORK__", "--execution-endpoint=http://geth_node:8551", "--execution-jwt=/var/lib/lighthouse/secrets/jwtsecret", - "--checkpoint-sync-url=https://sync-mainnet.beaconcha.in", + "--checkpoint-sync-url=__ETH_CONSENSUS_CHECKPOINT_SYNC_URL__", "--http", "--http-port=5052", "--http-address=0.0.0.0", diff --git a/lib/ethereum/lib/assets/docker-compose/docker-compose-nethermind-teku.yml b/lib/ethereum/lib/assets/node/docker-compose-nethermind-teku.yml similarity index 93% rename from lib/ethereum/lib/assets/docker-compose/docker-compose-nethermind-teku.yml rename to lib/ethereum/lib/assets/node/docker-compose-nethermind-teku.yml index 4b3d54ce..72ca15ab 100644 --- a/lib/ethereum/lib/assets/docker-compose/docker-compose-nethermind-teku.yml +++ b/lib/ethereum/lib/assets/node/docker-compose-nethermind-teku.yml @@ -9,7 +9,7 @@ services: restart: always command: [ "--config", - "mainnet", + "__ETH_NETWORK__", "--datadir", "/var/lib/nethermind/data", "--Sync.SnapSync", @@ -31,7 +31,7 @@ services: "--HealthChecks.Enabled", "true", ] - user: ethereum:ethereum + user: bcuser:bcuser volumes: - /secrets:/var/lib/nethermind/secrets - /data/execution/data:/var/lib/nethermind/data @@ -60,7 +60,7 @@ services: [ "--data-base-path=/var/lib/teku/data", "--ee-endpoint=http://nethermind_node:8551", - "--initial-state=https://beaconstate.info/eth/v2/debug/beacon/states/finalized", + "--checkpoint-sync-url=__ETH_CONSENSUS_CHECKPOINT_SYNC_URL__", "--ee-jwt-secret-file=/var/lib/teku/secrets/jwtsecret", "--p2p-port=9000", "--metrics-enabled=true", @@ -68,7 +68,7 @@ services: ] depends_on: - nethermind_node - user: ethereum:ethereum + user: bcuser:bcuser volumes: - /secrets:/var/lib/teku/secrets - /data/consensus/data:/var/lib/teku/data diff --git a/lib/ethereum/lib/assets/node/docker-compose-reth-lighthouse.yml b/lib/ethereum/lib/assets/node/docker-compose-reth-lighthouse.yml new file mode 100644 index 00000000..dbfe13b5 --- /dev/null +++ b/lib/ethereum/lib/assets/node/docker-compose-reth-lighthouse.yml @@ -0,0 +1,72 @@ +--- +version: "3" +services: + execution_node: + image: ghcr.io/paradigmxyz/reth + container_name: execution + restart: always + pid: host + command: > + node + --chain __ETH_NETWORK__ + --authrpc.addr 0.0.0.0 + --authrpc.port 8551 + --authrpc.jwtsecret /root/jwt/jwtsecret + --datadir /root/.local/share/reth + --http --http.addr 0.0.0.0 --http.port 8545 + --log.file.max-files 0 + --http.api "admin,eth,web3,txpool,net,debug,trace,reth" + --ws + --ws.port 8546 + + volumes: + - /secrets:/root/jwt + - /data/execution/data:/root/.local/share/reth + stop_signal: SIGTERM + stop_grace_period: 8m + security_opt: + - no-new-privileges:true + read_only: true + ports: + # Map the p2p port(30303), RPC HTTP port(8545), RPC WS port (8546), and engine port (8551) + - "8545:8545" + - "8546:8546" + - "8551:8551" + - "30303:30303/tcp" + - "30303:30303/udp" + + consensus_node: + image: sigp/lighthouse:v6.0.1 + container_name: consensus + restart: always + command: + [ + "lighthouse", + "beacon", + "--network=__ETH_NETWORK__", + "--execution-endpoint=http://execution_node:8551", + "--execution-jwt=/var/lib/lighthouse/secrets/jwtsecret", + "--checkpoint-sync-url=__ETH_CONSENSUS_CHECKPOINT_SYNC_URL__", + "--http", + "--http-port=5052", + "--http-address=0.0.0.0", + "--metrics", + "--datadir=/var/lib/lighthouse/data", + "--disable-deposit-contract-sync", + ] + depends_on: + - execution_node + volumes: + - /secrets:/var/lib/lighthouse/secrets + - /data/consensus/data:/var/lib/lighthouse/data + stop_signal: SIGTERM + stop_grace_period: 5m + security_opt: + - no-new-privileges:true + read_only: true + ports: + - 5052:5052/tcp + - 5053:5053/tcp + - 5054:5054/tcp # metrics endpoint + - 9000:9000/tcp + - 9000:9000/udp diff --git a/lib/ethereum/lib/assets/sync-checker/setup.sh b/lib/ethereum/lib/assets/sync-checker/setup.sh new file mode 100755 index 00000000..96bc10e5 --- /dev/null +++ b/lib/ethereum/lib/assets/sync-checker/setup.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +if [ -n "$1" ]; then + export SYNC_CHECKER_SCRIPT=$1 +else + echo "No path to syncchecker script is provided" + echo "Usage: sync-checker/setup.sh " + echo "Using default: /opt/sync-checker/syncchecker.sh" + export SYNC_CHECKER_SCRIPT="/opt/sync-checker/syncchecker.sh" +fi + +echo "Configuring syncchecker script" +mv $SYNC_CHECKER_SCRIPT /opt/syncchecker.sh +chmod +x /opt/syncchecker.sh + +echo "Setting up sync-checker service" +mv /opt/sync-checker/sync-checker.service /etc/systemd/system/sync-checker.service + +# Run every 5 minutes +echo "Setting up sync-checker timer" +mv /opt/sync-checker/sync-checker.timer /etc/systemd/system/sync-checker.timer + +echo "Starting sync checker timer" +systemctl start sync-checker.timer +systemctl enable sync-checker.timer diff --git a/lib/ethereum/lib/assets/sync-checker/sync-checker.service b/lib/ethereum/lib/assets/sync-checker/sync-checker.service new file mode 100644 index 00000000..9f187ce2 --- /dev/null +++ b/lib/ethereum/lib/assets/sync-checker/sync-checker.service @@ -0,0 +1,5 @@ +[Unit] +Description="Sync checker for blockchain node" + +[Service] +ExecStart=/opt/syncchecker.sh diff --git a/lib/ethereum/lib/assets/sync-checker/sync-checker.timer b/lib/ethereum/lib/assets/sync-checker/sync-checker.timer new file mode 100644 index 00000000..b45ff94e --- /dev/null +++ b/lib/ethereum/lib/assets/sync-checker/sync-checker.timer @@ -0,0 +1,9 @@ +[Unit] +Description="Run Sync checker service every 5 min" + +[Timer] +OnCalendar=*:*:0/5 +Unit=sync-checker.service + +[Install] +WantedBy=multi-user.target diff --git a/lib/ethereum/lib/assets/sync-checker/syncchecker-besu-teku.sh b/lib/ethereum/lib/assets/sync-checker/syncchecker-besu-teku.sh index e40f04f1..e38496e4 100644 --- a/lib/ethereum/lib/assets/sync-checker/syncchecker-besu-teku.sh +++ b/lib/ethereum/lib/assets/sync-checker/syncchecker-besu-teku.sh @@ -1,5 +1,5 @@ #!/bin/bash -source /etc/environment +source /etc/cdk_environment # Consensus client stats CONSENSUS_CLIENT_SYNC_STATUS=$(curl -s http://localhost:5052/eth/v1/node/syncing | jq -r ".data") @@ -52,10 +52,10 @@ aws cloudwatch put-metric-data --metric-name elc_blocks_behind --namespace CWAge if [[ "$NODE_ROLE" == "sync-node" ]]; then if [ ! -f "/data/snapshotted" ]; then if [ "$EXECUTION_CLIENT_SYNC_STATS" == "false" ] && [ "$CONSENSUS_CLIENT_IS_SYNCING" == "false" ] && [ "$CONSENSUS_CLIENT_IS_OPTIMISTIC" == "false" ]; then - sudo /opt/copy-data-to-s3.sh + sudo /opt/instance/storage/copy-data-to-s3.sh # Take a snapshot once a day at midnight - (sudo crontab -u root -l; echo '0 0 * * * /opt/copy-data-to-s3.sh' ) | sudo crontab -u root - + (sudo crontab -u root -l; echo '0 0 * * * /opt/instance/storage/copy-data-to-s3.sh' ) | sudo crontab -u root - sudo crontab -l fi fi diff --git a/lib/ethereum/lib/assets/sync-checker/syncchecker-erigon-caplin.sh b/lib/ethereum/lib/assets/sync-checker/syncchecker-erigon-caplin.sh new file mode 100644 index 00000000..2f4f21ac --- /dev/null +++ b/lib/ethereum/lib/assets/sync-checker/syncchecker-erigon-caplin.sh @@ -0,0 +1,62 @@ +#!/bin/bash +source /etc/cdk_environment + +# Consensus client stats +CONSENSUS_CLIENT_SYNC_STATUS=$(curl -s http://localhost:5052/eth/v1/node/syncing | jq -r ".data") + +CONSENSUS_CLIENT_IS_SYNCING=$(echo $CONSENSUS_CLIENT_SYNC_STATUS | jq -r ".is_syncing") +CONSENSUS_CLIENT_IS_OPTIMISTIC=$(echo $CONSENSUS_CLIENT_SYNC_STATUS | jq -r ".is_optimistic") +CONSENSUS_CLIENT_SYNC_DISTANCE=$(echo $CONSENSUS_CLIENT_SYNC_STATUS | jq -r ".sync_distance") +CONSENSUS_CLIENT_HEAD_SLOT=$(echo $CONSENSUS_CLIENT_SYNC_STATUS | jq -r ".head_slot") + +if [[ -z "$CONSENSUS_CLIENT_SYNC_DISTANCE" ]]; then + CONSENSUS_CLIENT_SYNC_DISTANCE=0 +fi + +if [[ -z "$CONSENSUS_CLIENT_HEAD_SLOT" ]]; then + CONSENSUS_CLIENT_HEAD_SLOT=0 +fi + +# Execution client stats +EXECUTION_CLIENT_SYNC_STATS=$(curl -s -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"eth_syncing","params":[],"id":1}' http://localhost:8545 | jq -r ".result") + +if [[ "$EXECUTION_CLIENT_SYNC_STATS" == "false" ]]; then + EXECUTION_CLIENT_SYNC_BLOCK_HEX=$(curl -s -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' http://localhost:8545 | jq -r ".result") + EXECUTION_CLIENT_HIGHEST_BLOCK_HEX=$EXECUTION_CLIENT_SYNC_BLOCK_HEX +else + EXECUTION_CLIENT_SYNC_BLOCK_HEX=$(echo $EXECUTION_CLIENT_SYNC_STATS | jq -r ".currentBlock") + EXECUTION_CLIENT_HIGHEST_BLOCK_HEX=$(echo $EXECUTION_CLIENT_SYNC_STATS | jq -r ".highestBlock") +fi + +EXECUTION_CLIENT_HIGHEST_BLOCK=$(echo $((${EXECUTION_CLIENT_HIGHEST_BLOCK_HEX}))) +EXECUTION_CLIENT_SYNC_BLOCK=$(echo $((${EXECUTION_CLIENT_SYNC_BLOCK_HEX}))) +EXECUTION_CLIENT_BLOCKS_BEHIND="$((EXECUTION_CLIENT_HIGHEST_BLOCK-EXECUTION_CLIENT_SYNC_BLOCK))" + +echo "EXECUTION_CLIENT_SYNC_STATS="$EXECUTION_CLIENT_SYNC_STATS +echo "CONSENSUS_CLIENT_IS_SYNCING="$CONSENSUS_CLIENT_IS_SYNCING +echo "CONSENSUS_CLIENT_IS_OPTIMISTIC="$CONSENSUS_CLIENT_IS_OPTIMISTIC + +# Sending data to CloudWatch +TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") +INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/instance-id) +REGION=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq .region -r) +TIMESTAMP=$(date +"%Y-%m-%dT%H:%M:%S%:z") + +aws cloudwatch put-metric-data --metric-name clc_sync_distance --namespace CWAgent --value $CONSENSUS_CLIENT_SYNC_DISTANCE --timestamp $TIMESTAMP --dimensions InstanceId=$INSTANCE_ID --region $REGION +aws cloudwatch put-metric-data --metric-name clc_head_slot --namespace CWAgent --value $CONSENSUS_CLIENT_HEAD_SLOT --timestamp $TIMESTAMP --dimensions InstanceId=$INSTANCE_ID --region $REGION + +aws cloudwatch put-metric-data --metric-name elc_sync_block --namespace CWAgent --value $EXECUTION_CLIENT_SYNC_BLOCK --timestamp $TIMESTAMP --dimensions InstanceId=$INSTANCE_ID --region $REGION +aws cloudwatch put-metric-data --metric-name elc_blocks_behind --namespace CWAgent --value $EXECUTION_CLIENT_BLOCKS_BEHIND --timestamp $TIMESTAMP --dimensions InstanceId=$INSTANCE_ID --region $REGION + +# If the node is a sync node, check if the snapshot is already taken. If the snapshot is not taken, then take it and restart the node. +if [[ "$NODE_ROLE" == "sync-node" ]]; then + if [ ! -f "/data/snapshotted" ]; then + if [ "$EXECUTION_CLIENT_SYNC_STATS" == "false" ] && [ "$CONSENSUS_CLIENT_IS_SYNCING" == "false" ] && [ "$CONSENSUS_CLIENT_IS_OPTIMISTIC" == "false" ]; then + sudo /opt/instance/storage/copy-data-to-s3.sh + + # Take a snapshot once a day at midnight + (sudo crontab -u root -l; echo '0 0 * * * /opt/instance/storage/copy-data-to-s3.sh' ) | sudo crontab -u root - + sudo crontab -l + fi + fi +fi diff --git a/lib/ethereum/lib/assets/sync-checker/syncchecker-erigon-lighthouse.sh b/lib/ethereum/lib/assets/sync-checker/syncchecker-erigon-lighthouse.sh index 5450f1c8..2f4f21ac 100644 --- a/lib/ethereum/lib/assets/sync-checker/syncchecker-erigon-lighthouse.sh +++ b/lib/ethereum/lib/assets/sync-checker/syncchecker-erigon-lighthouse.sh @@ -1,5 +1,5 @@ #!/bin/bash -source /etc/environment +source /etc/cdk_environment # Consensus client stats CONSENSUS_CLIENT_SYNC_STATUS=$(curl -s http://localhost:5052/eth/v1/node/syncing | jq -r ".data") @@ -52,10 +52,10 @@ aws cloudwatch put-metric-data --metric-name elc_blocks_behind --namespace CWAge if [[ "$NODE_ROLE" == "sync-node" ]]; then if [ ! -f "/data/snapshotted" ]; then if [ "$EXECUTION_CLIENT_SYNC_STATS" == "false" ] && [ "$CONSENSUS_CLIENT_IS_SYNCING" == "false" ] && [ "$CONSENSUS_CLIENT_IS_OPTIMISTIC" == "false" ]; then - sudo /opt/copy-data-to-s3.sh + sudo /opt/instance/storage/copy-data-to-s3.sh # Take a snapshot once a day at midnight - (sudo crontab -u root -l; echo '0 0 * * * /opt/copy-data-to-s3.sh' ) | sudo crontab -u root - + (sudo crontab -u root -l; echo '0 0 * * * /opt/instance/storage/copy-data-to-s3.sh' ) | sudo crontab -u root - sudo crontab -l fi fi diff --git a/lib/ethereum/lib/assets/sync-checker/syncchecker-erigon-prysm.sh b/lib/ethereum/lib/assets/sync-checker/syncchecker-erigon-prysm.sh index 5450f1c8..2f4f21ac 100644 --- a/lib/ethereum/lib/assets/sync-checker/syncchecker-erigon-prysm.sh +++ b/lib/ethereum/lib/assets/sync-checker/syncchecker-erigon-prysm.sh @@ -1,5 +1,5 @@ #!/bin/bash -source /etc/environment +source /etc/cdk_environment # Consensus client stats CONSENSUS_CLIENT_SYNC_STATUS=$(curl -s http://localhost:5052/eth/v1/node/syncing | jq -r ".data") @@ -52,10 +52,10 @@ aws cloudwatch put-metric-data --metric-name elc_blocks_behind --namespace CWAge if [[ "$NODE_ROLE" == "sync-node" ]]; then if [ ! -f "/data/snapshotted" ]; then if [ "$EXECUTION_CLIENT_SYNC_STATS" == "false" ] && [ "$CONSENSUS_CLIENT_IS_SYNCING" == "false" ] && [ "$CONSENSUS_CLIENT_IS_OPTIMISTIC" == "false" ]; then - sudo /opt/copy-data-to-s3.sh + sudo /opt/instance/storage/copy-data-to-s3.sh # Take a snapshot once a day at midnight - (sudo crontab -u root -l; echo '0 0 * * * /opt/copy-data-to-s3.sh' ) | sudo crontab -u root - + (sudo crontab -u root -l; echo '0 0 * * * /opt/instance/storage/copy-data-to-s3.sh' ) | sudo crontab -u root - sudo crontab -l fi fi diff --git a/lib/ethereum/lib/assets/sync-checker/syncchecker-geth-lighthouse.sh b/lib/ethereum/lib/assets/sync-checker/syncchecker-geth-lighthouse.sh index d85f1bde..0662d7c1 100644 --- a/lib/ethereum/lib/assets/sync-checker/syncchecker-geth-lighthouse.sh +++ b/lib/ethereum/lib/assets/sync-checker/syncchecker-geth-lighthouse.sh @@ -1,5 +1,5 @@ #!/bin/bash -source /etc/environment +source /etc/cdk_environment # Consensus client stats CONSENSUS_CLIENT_SYNC_STATUS=$(curl -s http://localhost:5052/eth/v1/node/syncing | jq -r ".data") @@ -55,10 +55,10 @@ aws cloudwatch put-metric-data --metric-name elc_blocks_behind --namespace CWAge if [[ "$NODE_ROLE" == "sync-node" ]]; then if [ ! -f "/data/snapshotted" ]; then if [ "$EXECUTION_CLIENT_SYNC_STATS" == "false" ] && [ "$CONSENSUS_CLIENT_IS_SYNCING" == "false" ] && [ "$CONSENSUS_CLIENT_IS_OPTIMISTIC" == "false" ]; then - sudo /opt/copy-data-to-s3.sh + sudo /opt/instance/storage/copy-data-to-s3.sh # Take a snapshot once a day at midnight - (sudo crontab -u root -l; echo '0 0 * * * /opt/copy-data-to-s3.sh' ) | sudo crontab -u root - + (sudo crontab -u root -l; echo '0 0 * * * /opt/instance/storage/copy-data-to-s3.sh' ) | sudo crontab -u root - sudo crontab -l fi fi diff --git a/lib/ethereum/lib/assets/sync-checker/syncchecker-nethermind-teku.sh b/lib/ethereum/lib/assets/sync-checker/syncchecker-nethermind-teku.sh index e40f04f1..e38496e4 100644 --- a/lib/ethereum/lib/assets/sync-checker/syncchecker-nethermind-teku.sh +++ b/lib/ethereum/lib/assets/sync-checker/syncchecker-nethermind-teku.sh @@ -1,5 +1,5 @@ #!/bin/bash -source /etc/environment +source /etc/cdk_environment # Consensus client stats CONSENSUS_CLIENT_SYNC_STATUS=$(curl -s http://localhost:5052/eth/v1/node/syncing | jq -r ".data") @@ -52,10 +52,10 @@ aws cloudwatch put-metric-data --metric-name elc_blocks_behind --namespace CWAge if [[ "$NODE_ROLE" == "sync-node" ]]; then if [ ! -f "/data/snapshotted" ]; then if [ "$EXECUTION_CLIENT_SYNC_STATS" == "false" ] && [ "$CONSENSUS_CLIENT_IS_SYNCING" == "false" ] && [ "$CONSENSUS_CLIENT_IS_OPTIMISTIC" == "false" ]; then - sudo /opt/copy-data-to-s3.sh + sudo /opt/instance/storage/copy-data-to-s3.sh # Take a snapshot once a day at midnight - (sudo crontab -u root -l; echo '0 0 * * * /opt/copy-data-to-s3.sh' ) | sudo crontab -u root - + (sudo crontab -u root -l; echo '0 0 * * * /opt/instance/storage/copy-data-to-s3.sh' ) | sudo crontab -u root - sudo crontab -l fi fi diff --git a/lib/ethereum/lib/assets/sync-checker/syncchecker-reth-lighthouse.sh b/lib/ethereum/lib/assets/sync-checker/syncchecker-reth-lighthouse.sh new file mode 100644 index 00000000..2f4f21ac --- /dev/null +++ b/lib/ethereum/lib/assets/sync-checker/syncchecker-reth-lighthouse.sh @@ -0,0 +1,62 @@ +#!/bin/bash +source /etc/cdk_environment + +# Consensus client stats +CONSENSUS_CLIENT_SYNC_STATUS=$(curl -s http://localhost:5052/eth/v1/node/syncing | jq -r ".data") + +CONSENSUS_CLIENT_IS_SYNCING=$(echo $CONSENSUS_CLIENT_SYNC_STATUS | jq -r ".is_syncing") +CONSENSUS_CLIENT_IS_OPTIMISTIC=$(echo $CONSENSUS_CLIENT_SYNC_STATUS | jq -r ".is_optimistic") +CONSENSUS_CLIENT_SYNC_DISTANCE=$(echo $CONSENSUS_CLIENT_SYNC_STATUS | jq -r ".sync_distance") +CONSENSUS_CLIENT_HEAD_SLOT=$(echo $CONSENSUS_CLIENT_SYNC_STATUS | jq -r ".head_slot") + +if [[ -z "$CONSENSUS_CLIENT_SYNC_DISTANCE" ]]; then + CONSENSUS_CLIENT_SYNC_DISTANCE=0 +fi + +if [[ -z "$CONSENSUS_CLIENT_HEAD_SLOT" ]]; then + CONSENSUS_CLIENT_HEAD_SLOT=0 +fi + +# Execution client stats +EXECUTION_CLIENT_SYNC_STATS=$(curl -s -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"eth_syncing","params":[],"id":1}' http://localhost:8545 | jq -r ".result") + +if [[ "$EXECUTION_CLIENT_SYNC_STATS" == "false" ]]; then + EXECUTION_CLIENT_SYNC_BLOCK_HEX=$(curl -s -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' http://localhost:8545 | jq -r ".result") + EXECUTION_CLIENT_HIGHEST_BLOCK_HEX=$EXECUTION_CLIENT_SYNC_BLOCK_HEX +else + EXECUTION_CLIENT_SYNC_BLOCK_HEX=$(echo $EXECUTION_CLIENT_SYNC_STATS | jq -r ".currentBlock") + EXECUTION_CLIENT_HIGHEST_BLOCK_HEX=$(echo $EXECUTION_CLIENT_SYNC_STATS | jq -r ".highestBlock") +fi + +EXECUTION_CLIENT_HIGHEST_BLOCK=$(echo $((${EXECUTION_CLIENT_HIGHEST_BLOCK_HEX}))) +EXECUTION_CLIENT_SYNC_BLOCK=$(echo $((${EXECUTION_CLIENT_SYNC_BLOCK_HEX}))) +EXECUTION_CLIENT_BLOCKS_BEHIND="$((EXECUTION_CLIENT_HIGHEST_BLOCK-EXECUTION_CLIENT_SYNC_BLOCK))" + +echo "EXECUTION_CLIENT_SYNC_STATS="$EXECUTION_CLIENT_SYNC_STATS +echo "CONSENSUS_CLIENT_IS_SYNCING="$CONSENSUS_CLIENT_IS_SYNCING +echo "CONSENSUS_CLIENT_IS_OPTIMISTIC="$CONSENSUS_CLIENT_IS_OPTIMISTIC + +# Sending data to CloudWatch +TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") +INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/instance-id) +REGION=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq .region -r) +TIMESTAMP=$(date +"%Y-%m-%dT%H:%M:%S%:z") + +aws cloudwatch put-metric-data --metric-name clc_sync_distance --namespace CWAgent --value $CONSENSUS_CLIENT_SYNC_DISTANCE --timestamp $TIMESTAMP --dimensions InstanceId=$INSTANCE_ID --region $REGION +aws cloudwatch put-metric-data --metric-name clc_head_slot --namespace CWAgent --value $CONSENSUS_CLIENT_HEAD_SLOT --timestamp $TIMESTAMP --dimensions InstanceId=$INSTANCE_ID --region $REGION + +aws cloudwatch put-metric-data --metric-name elc_sync_block --namespace CWAgent --value $EXECUTION_CLIENT_SYNC_BLOCK --timestamp $TIMESTAMP --dimensions InstanceId=$INSTANCE_ID --region $REGION +aws cloudwatch put-metric-data --metric-name elc_blocks_behind --namespace CWAgent --value $EXECUTION_CLIENT_BLOCKS_BEHIND --timestamp $TIMESTAMP --dimensions InstanceId=$INSTANCE_ID --region $REGION + +# If the node is a sync node, check if the snapshot is already taken. If the snapshot is not taken, then take it and restart the node. +if [[ "$NODE_ROLE" == "sync-node" ]]; then + if [ ! -f "/data/snapshotted" ]; then + if [ "$EXECUTION_CLIENT_SYNC_STATS" == "false" ] && [ "$CONSENSUS_CLIENT_IS_SYNCING" == "false" ] && [ "$CONSENSUS_CLIENT_IS_OPTIMISTIC" == "false" ]; then + sudo /opt/instance/storage/copy-data-to-s3.sh + + # Take a snapshot once a day at midnight + (sudo crontab -u root -l; echo '0 0 * * * /opt/instance/storage/copy-data-to-s3.sh' ) | sudo crontab -u root - + sudo crontab -l + fi + fi +fi diff --git a/lib/ethereum/lib/assets/user-data-alinux.sh b/lib/ethereum/lib/assets/user-data-alinux.sh new file mode 100644 index 00000000..49eccddc --- /dev/null +++ b/lib/ethereum/lib/assets/user-data-alinux.sh @@ -0,0 +1,214 @@ +#!/bin/bash +set +e + +touch /etc/cdk_environment +chmod 600 /etc/cdk_environment + +{ + echo "AWS_REGION=${_REGION_}" + echo "ETH_SNAPSHOT_TYPE=${_ETH_SNAPSHOT_TYPE_}" + echo "SNAPSHOT_S3_PATH=${_SNAPSHOT_S3_PATH_}" + echo "ETH_CLIENT_COMBINATION=${_ETH_CLIENT_COMBINATION_}" + echo "ETH_NETWORK=${_ETH_NETWORK_}" + echo "ETH_CONSENSUS_CHECKPOINT_SYNC_URL=${_ETH_CONSENSUS_CHECKPOINT_SYNC_URL_}" + echo "STACK_NAME=${_STACK_NAME_}" + echo "AUTOSTART_CONTAINER=${_AUTOSTART_CONTAINER_}" + echo "FORMAT_DISK=${_FORMAT_DISK_}" + echo "DATA_VOLUME_TYPE=${_DATA_VOLUME_TYPE_}" + echo "DATA_VOLUME_SIZE=${_DATA_VOLUME_SIZE_}" + echo "NODE_ROLE=${_NODE_ROLE_}" + echo "RESOURCE_ID=${_NODE_CF_LOGICAL_ID_}" + echo "LIFECYCLE_HOOK_NAME=${_LIFECYCLE_HOOK_NAME_}" + echo "AUTOSCALING_GROUP_NAME=${_AUTOSCALING_GROUP_NAME_}" + echo "ASSETS_S3_PATH=${_ASSETS_S3_PATH_}" +} >> /etc/cdk_environment +source /etc/cdk_environment + +# Export environment variables so calls to `envsubst` inherit the evironment variables. +while read -r line; do export "$line"; done < /etc/cdk_environment + +arch=$(uname -m) + +echo "Architecture detected: $arch" + +if [ "$arch" == "x86_64" ]; then + SSM_AGENT_BINARY_URI=https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm + S5CMD_URI=https://github.com/peak/s5cmd/releases/download/v2.1.0/s5cmd_2.1.0_Linux-64bit.tar.gz +else + SSM_AGENT_BINARY_URI=https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_arm64/amazon-ssm-agent.rpm + S5CMD_URI=https://github.com/peak/s5cmd/releases/download/v2.1.0/s5cmd_2.1.0_Linux-arm64.tar.gz +fi + +echo "Updating and installing required system packages" +dnf update -y +dnf -y install amazon-cloudwatch-agent collectd jq gcc ncurses-devel telnet aws-cfn-bootstrap cronie + +sudo systemctl enable crond.service +sudo systemctl start crond.service + +cd /opt + +echo "Downloading assets zip file" +aws s3 cp $ASSETS_S3_PATH ./assets.zip +unzip -q assets.zip + +aws configure set default.s3.max_concurrent_requests 50 +aws configure set default.s3.multipart_chunksize 256MB + +echo 'Upgrading SSM Agent' +yum install -y $SSM_AGENT_BINARY_URI + +echo "Installing s5cmd" +cd /opt +wget -q $S5CMD_URI -O s5cmd.tar.gz +tar -xf s5cmd.tar.gz +chmod +x s5cmd +mv s5cmd /usr/bin +s5cmd version + +# Ethereum specific setup starts here + +echo "Ethereum Client combination: $ETH_CLIENT_COMBINATION" + +# Can add more combination in the future +case $ETH_CLIENT_COMBINATION in + "besu-teku") + SYNC_CHECKER_FILE_NAME=syncchecker-besu-teku.sh + DOCKER_COMPOSE_FILE_NAME=docker-compose-besu-teku.yml + ;; + "geth-lighthouse") + SYNC_CHECKER_FILE_NAME=syncchecker-geth-lighthouse.sh + DOCKER_COMPOSE_FILE_NAME=docker-compose-geth-lighthouse.yml + ;; + "erigon-lighthouse") + SYNC_CHECKER_FILE_NAME=syncchecker-erigon-lighthouse.sh + DOCKER_COMPOSE_FILE_NAME=docker-compose-erigon-lighthouse.yml + ;; + "erigon-prysm") + SYNC_CHECKER_FILE_NAME=syncchecker-erigon-prysm.sh + DOCKER_COMPOSE_FILE_NAME=docker-compose-erigon-prysm.yml + ;; + "nethermind-teku") + SYNC_CHECKER_FILE_NAME=syncchecker-nethermind-teku.sh + DOCKER_COMPOSE_FILE_NAME=docker-compose-nethermind-teku.yml + ;; + "reth-lighthouse") + SYNC_CHECKER_FILE_NAME=syncchecker-reth-lighthouse.sh + DOCKER_COMPOSE_FILE_NAME=docker-compose-reth-lighthouse.yml + ;; + *) + echo "Combination is not valid." + exit 1 + ;; +esac + +echo "Installing Docker" +dnf remove -y docker docker-client docker-client-latest docker-common docker-latest docker-latest-logrotate docker-logrotate docker-engine +dnf -y install dnf-plugins-core +dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo +sed -i 's/$releasever/9/g' /etc/yum.repos.d/docker-ce.repo +dnf -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +systemctl enable --now docker + +echo 'Preparing secrets' +mkdir -p /secrets +openssl rand -hex 32 | tr -d "\n" | sudo tee /secrets/jwtsecret + +echo "Creating run user and making sure it has all necessary permissions" +groupadd -g 1002 bcuser +useradd -u 1002 -g 1002 -m -s /bin/bash bcuser +usermod -a -G docker bcuser +usermod -a -G docker ec2-user +chown -R bcuser:bcuser /secrets +chmod -R 755 /home/bcuser +chmod -R 755 /secrets + +echo "Starting docker" +service docker start +systemctl enable docker + +echo "Copying docker-compose file" +cp ./node/$DOCKER_COMPOSE_FILE_NAME /home/bcuser/docker-compose.yml + +sed -i 's/__ETH_NETWORK__/'"$ETH_NETWORK"'/g' /home/bcuser/docker-compose.yml +sed -i 's,__ETH_CONSENSUS_CHECKPOINT_SYNC_URL__,'"$ETH_CONSENSUS_CHECKPOINT_SYNC_URL"',g' /home/bcuser/docker-compose.yml +chown -R bcuser:bcuser /home/bcuser/docker-compose.yml + +echo "Configuring and starting sync-checker" +/opt/sync-checker/setup.sh "/opt/sync-checker/$SYNC_CHECKER_FILE_NAME" + +# If in Single Node stack (have Stack ID), configuring ClodFormation helpers to signal the completion of deployment" +if [[ "$STACK_ID" != "none" ]]; then + #If cfn-signal is not available, install it + if ! command -v cfn-signal &> /dev/null + then + echo "cfn-signal could not be found, installing" + /opt/instance/cfn-hup/setup.sh "$STACK_NAME" "$AWS_REGION" + else + echo "cfn-signal is available, skipping installation" + fi + cfn-signal --stack "$STACK_NAME" --resource "$RESOURCE_ID" --region "$AWS_REGION" +fi + +echo "Preparing data volume" + +mkdir -p /data + +if [[ "$DATA_VOLUME_TYPE" == "instance-store" ]]; then + echo "Data volume type is instance store" + + (crontab -l; echo "@reboot /opt/instance/storage/setup.sh /data ext4 > /tmp/setup-store-volume-data.log 2>&1") | crontab - + crontab -l + + /opt/instance/storage/setup.sh /data ext4 +else + echo "Data volume type is EBS" + echo "Waiting for EBS volume to become available" + sleep 60 + /opt/instance/storage/setup.sh /data ext4 +fi + +lsblk -d + +mkdir -p /data/execution +mkdir -p /data/consensus +mkdir -p /data/execution/data +mkdir -p /data/execution/others +mkdir -p /data/consensus/data + +chown -R bcuser:bcuser /data +chmod -R 755 /data + +echo 'Configuring CloudWatch Agent' +cp /opt/cw-agent.json /opt/aws/amazon-cloudwatch-agent/etc/custom-amazon-cloudwatch-agent.json + +echo "Starting CloudWatch Agent" +/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ +-a fetch-config -c file:/opt/aws/amazon-cloudwatch-agent/etc/custom-amazon-cloudwatch-agent.json -m ec2 -s +systemctl restart amazon-cloudwatch-agent + +if [ "$NODE_ROLE" == "sync-node" ] || [ "$NODE_ROLE" == "single-node" ]; then + if [ "$AUTOSTART_CONTAINER" == "false" ]; then + echo "Single node. Autostart disabled. Start docker-compose manually!" + else + if [ "$ETH_SNAPSHOT_TYPE" == "none" ]; then + echo "Snapshot is not provided. Autostart enabled. Starting docker compose" + docker compose -f /home/bcuser/docker-compose.yml up -d + fi + fi +fi + +if [ "$NODE_ROLE" == "sync-node" ]; then + echo "Sync node. Configuring snapshotting script." + chmod 766 /opt/instance/storage/copy-data-to-s3.sh +fi + +if [ "$NODE_ROLE" == "rpc-node" ] || [ "$NODE_ROLE" == "single-node" ]; then + if [ "$ETH_SNAPSHOT_TYPE" == "s3" ]; then + echo "RPC node. Snapshot on S3. Starting copy data script" + chmod 766 /opt/instance/storage/copy-data-from-s3.sh + echo "/opt/instance/storage/copy-data-from-s3.sh" | at now +3 minutes + fi +fi + +echo "All Done!!" diff --git a/lib/ethereum/lib/assets/user-data/node.sh b/lib/ethereum/lib/assets/user-data/node.sh deleted file mode 100644 index d41dc133..00000000 --- a/lib/ethereum/lib/assets/user-data/node.sh +++ /dev/null @@ -1,260 +0,0 @@ -#!/bin/bash -set +e - -# Set by generic single-node and ha-node CDK components -LIFECYCLE_HOOK_NAME=${_LIFECYCLE_HOOK_NAME_} -AUTOSCALING_GROUP_NAME=${_AUTOSCALING_GROUP_NAME_} -RESOURCE_ID=${_NODE_CF_LOGICAL_ID_} -ASSETS_S3_PATH=${_ASSETS_S3_PATH_} -echo "LIFECYCLE_HOOK_NAME=$LIFECYCLE_HOOK_NAME" >> /etc/environment -echo "AUTOSCALING_GROUP_NAME=$AUTOSCALING_GROUP_NAME" >> /etc/environment -echo "ASSETS_S3_PATH=$ASSETS_S3_PATH" >> /etc/environment - -arch=$(uname -m) - -echo "Architecture detected: $arch" - -if [ "$arch" == "x86_64" ]; then - SSM_AGENT_BINARY_URI=https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm - AWS_CLI_BINARY_URI=https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip - S5CMD_URI=https://github.com/peak/s5cmd/releases/download/v2.1.0/s5cmd_2.1.0_Linux-64bit.tar.gz -else - SSM_AGENT_BINARY_URI=https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_arm64/amazon-ssm-agent.rpm - AWS_CLI_BINARY_URI=https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip - S5CMD_URI=https://github.com/peak/s5cmd/releases/download/v2.1.0/s5cmd_2.1.0_Linux-arm64.tar.gz -fi - -echo "Updating and installing required system packages" -yum update -y -yum -y install amazon-cloudwatch-agent collectd jq gcc ncurses-devel telnet aws-cfn-bootstrap - -cd /opt - -echo "Downloading assets zip file" -aws s3 cp $ASSETS_S3_PATH ./assets.zip -unzip -q assets.zip - -echo 'Configuring CloudWatch Agent' -cp /opt/cw-agent.json /opt/aws/amazon-cloudwatch-agent/etc/custom-amazon-cloudwatch-agent.json - -echo "Starting CloudWatch Agent" -/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ --a fetch-config -c file:/opt/aws/amazon-cloudwatch-agent/etc/custom-amazon-cloudwatch-agent.json -m ec2 -s -systemctl status amazon-cloudwatch-agent - -echo 'Uninstalling AWS CLI v1' -yum remove awscli - -echo 'Installing AWS CLI v2' -curl $AWS_CLI_BINARY_URI -o "awscliv2.zip" -unzip -q awscliv2.zip -./aws/install -rm /usr/bin/aws -ln /usr/local/bin/aws /usr/bin/aws - -aws configure set default.s3.max_concurrent_requests 50 -aws configure set default.s3.multipart_chunksize 256MB - -echo 'Installing SSM Agent' -yum install -y $SSM_AGENT_BINARY_URI - -echo "Installing s5cmd" -cd /opt -wget -q $S5CMD_URI -O s5cmd.tar.gz -tar -xf s5cmd.tar.gz -chmod +x s5cmd -mv s5cmd /usr/bin -s5cmd version - -# Ethereum specific setup starts here - -# Set by Ethereum-specic CDK components and stacks -REGION=${_REGION_} -SNAPSHOT_S3_PATH=${_SNAPSHOT_S3_PATH_} -ETH_CLIENT_COMBINATION=${_ETH_CLIENT_COMBINATION_} -STACK_NAME=${_STACK_NAME_} -AUTOSTART_CONTAINER=${_AUTOSTART_CONTAINER_} -FORMAT_DISK=${_FORMAT_DISK_} -NODE_ROLE=${_NODE_ROLE_} - -echo "REGION=$REGION" >> /etc/environment -echo "SNAPSHOT_S3_PATH=$SNAPSHOT_S3_PATH" >> /etc/environment -echo "ETH_CLIENT_COMBINATION=$ETH_CLIENT_COMBINATION" >> /etc/environment -echo "NODE_ROLE=$NODE_ROLE" >> /etc/environment - -echo "Ethereum Client combination: $ETH_CLIENT_COMBINATION" - -# Can add more combination in the future -case $ETH_CLIENT_COMBINATION in - "besu-teku") - SYNC_CHECKER_FILE_NAME=syncchecker-besu-teku.sh - DOCKER_COMPOSE_FILE_NAME=docker-compose-besu-teku.yml - ;; - "geth-lighthouse") - SYNC_CHECKER_FILE_NAME=syncchecker-geth-lighthouse.sh - DOCKER_COMPOSE_FILE_NAME=docker-compose-geth-lighthouse.yml - ;; - "erigon-lighthouse") - SYNC_CHECKER_FILE_NAME=syncchecker-erigon-lighthouse.sh - DOCKER_COMPOSE_FILE_NAME=docker-compose-erigon-lighthouse.yml - ;; - "erigon-prysm") - SYNC_CHECKER_FILE_NAME=syncchecker-erigon-prysm.sh - DOCKER_COMPOSE_FILE_NAME=docker-compose-erigon-prysm.yml - ;; - "nethermind-teku") - SYNC_CHECKER_FILE_NAME=syncchecker-nethermind-teku.sh - DOCKER_COMPOSE_FILE_NAME=docker-compose-nethermind-teku.yml - ;; - *) - echo "Combination is not valid." - exit 1 - ;; -esac - -yum -y install docker python3-pip cronie cronie-anacron gcc python3-devel -yum -y remove python-requests -pip3 install docker-compose -pip3 install hapless -pip3 uninstall -y urllib3 -pip3 install 'urllib3<2.0' - -echo "Assigning Swap Space" -# Check if a swap file already exists -if [ -f /swapfile ]; then - # Remove the existing swap file - swapoff /swapfile - rm -rf /swapfile -fi - -# Create a new swap file -total_mem=$(grep MemTotal /proc/meminfo | awk '{print $2}') -# Calculate the swap size -swap_size=$((total_mem / 3)) -# Convert the swap size to MB -swap_size_mb=$((swap_size / 1024)) -unit=M -fallocate -l $swap_size_mb$unit /swapfile -chmod 600 /swapfile -mkswap /swapfile -swapon /swapfile - -# Enable the swap space to persist after reboot. -echo "/swapfile none swap sw 0 0" | sudo tee -a /etc/fstab - -sysctl vm.swappiness=6 -sysctl vm.vfs_cache_pressure=10 -echo "vm.swappiness=10" | sudo tee -a /etc/sysctl.conf -echo "vm.vfs_cache_pressure=10" | sudo tee -a /etc/sysctl.conf - -free -h - -echo 'Preparing secrets' -mkdir -p /secrets -openssl rand -hex 32 | tr -d "\n" | sudo tee /secrets/jwtsecret - -mkdir -p /data - -# Creating run user and making sure it has all necessary permissions -groupadd -g 1002 ethereum -useradd -u 1002 -g 1002 -m -s /bin/bash ethereum -usermod -a -G docker ethereum -usermod -a -G docker ec2-user -chown -R ethereum:ethereum /secrets -chmod -R 755 /home/ethereum -chmod -R 755 /secrets - -echo "Starting docker" -service docker start -systemctl enable docker - -echo "Copying docker-compose file" -cp ./docker-compose/$DOCKER_COMPOSE_FILE_NAME /home/ethereum/docker-compose.yml - -if [ "$ETH_CLIENT_COMBINATION" = "erigon-lighthouse" ] || [ "$ETH_CLIENT_COMBINATION" = "erigon-prysm" ]; then - echo "Configuring correct image architecture for Erigon" - if [ "$arch" = "x86_64" ]; then - sed -i 's/__IMAGE_ARCH__/amd64/g' /home/ethereum/docker-compose.yml - else - sed -i 's/__IMAGE_ARCH__/arm64/g' /home/ethereum/docker-compose.yml - fi - chown -R ethereum:ethereum /home/ethereum/docker-compose.yml -fi - -echo "Configuring syncchecker script" -cp /opt/sync-checker/$SYNC_CHECKER_FILE_NAME /opt/syncchecker.sh -chmod 766 /opt/syncchecker.sh - -echo "*/5 * * * * /opt/syncchecker.sh" | crontab -crontab -l - -if [ "$NODE_ROLE" == "sync-node" ]; then - echo "Sync node. Configuring snapshotting script." - chmod 766 /opt/copy-data-to-s3.sh -fi - -if [ "$NODE_ROLE" == "sync-node" ] || [ "$NODE_ROLE" == "single-node" ]; then - echo "Single node. Signaling completion to CloudFormation" - /opt/aws/bin/cfn-signal --stack $STACK_NAME --resource $RESOURCE_ID --region $REGION -fi - -echo "Preparing data volume" - -if [ "$NODE_ROLE" == "sync-node" ] || [ "$NODE_ROLE" == "single-node" ]; then - # echo "Sync node. Wait for the volume to be attached" - # aws ec2 wait volume-available --volume-ids $DATA_VOLUME_ID --region $REGION - - echo "Single node. Wait for one minute for the volume to be available" - sleep 60 -fi - -if $(lsblk | grep -q nvme1n1); then - echo "nvme1n1 is found. Configuring attached storage" - - if [ "$FORMAT_DISK" == "false" ]; then - echo "Not creating a new filesystem in the disk. Existing data might be present!!" - else - mkfs -t ext4 /dev/nvme1n1 - fi - - sleep 10 - # Define the line to add to fstab - uuid=$(lsblk -n -o UUID /dev/nvme1n1) - line="UUID=$uuid /data ext4 defaults 0 2" - - # Write the line to fstab - echo $line | sudo tee -a /etc/fstab - - mount -a - -else - echo "nvme1n1 is not found. Not doing anything" -fi - -lsblk -d - -mkdir -p /data/execution -mkdir -p /data/consensus -mkdir -p /data/execution/data -mkdir -p /data/execution/others -mkdir -p /data/consensus/data - -chown -R ethereum:ethereum /data -chmod -R 755 /data - -if [ "$NODE_ROLE" == "sync-node" ] || [ "$NODE_ROLE" == "single-node" ]; then - if [ "$AUTOSTART_CONTAINER" == "false" ]; then - echo "Single node. Autostart disabled. Start docker-compose manually!" - else - echo "Single node. Autostart enabled. Starting docker-compose in 3 min." - echo "sudo su ethereum && /usr/local/bin/docker-compose -f /home/ethereum/docker-compose.yml up -d" | at now +3 minutes - fi -fi - -if [ "$NODE_ROLE" == "rpc-node" ]; then - echo "RPC node. Starting copy data script" - chmod 766 /opt/copy-data-from-s3.sh - echo "/opt/copy-data-from-s3.sh" | at now +3 minutes -fi - -echo "All Done!!" diff --git a/lib/ethereum/lib/common-stack.ts b/lib/ethereum/lib/common-stack.ts index 067e6466..4a49e430 100644 --- a/lib/ethereum/lib/common-stack.ts +++ b/lib/ethereum/lib/common-stack.ts @@ -5,9 +5,12 @@ import * as iam from "aws-cdk-lib/aws-iam"; import * as nag from "cdk-nag"; import * as s3Assets from "aws-cdk-lib/aws-s3-assets"; import * as path from "path"; +import * as configTypes from "./config/node-config.interface"; import { SnapshotsS3BucketConstruct } from "../../constructs/snapshots-bucket"; -export interface EthCommonStackProps extends cdk.StackProps {} +export interface EthCommonStackProps extends cdk.StackProps { + snapshotType: configTypes.SnapshotType; +} export class EthCommonStack extends cdk.Stack { AWS_STACKNAME = cdk.Stack.of(this).stackName; @@ -23,14 +26,6 @@ export class EthCommonStack extends cdk.Stack { path: path.join(__dirname, "assets"), }); - const snapshotsBucket = new SnapshotsS3BucketConstruct(this, `snapshots-s3-bucket`, { - bucketName: `eth-snapshots-${this.AWS_STACKNAME}-${this.AWS_ACCOUNT_ID}-${this.AWS_REGION}`, - }); - - const s3VPCEndpoint = vpc.addGatewayEndpoint("s3-vpc-endpoint", { - service: ec2.GatewayVpcEndpointAwsService.S3, - }); - const instanceRole = new iam.Role(this, `node-role`, { assumedBy: new iam.ServicePrincipal("ec2.amazonaws.com"), managedPolicies: [ @@ -56,54 +51,65 @@ export class EthCommonStack extends cdk.Stack { }) ); - instanceRole.addToPolicy( - new iam.PolicyStatement({ - resources: [snapshotsBucket.bucketArn, snapshotsBucket.arnForObjects("*")], - actions: ["s3:ListBucket", "s3:*Object"], - }) - ); - - // Limiting access through this VPC endpoint only to our sync bucket and Amazon linux repo bucket - s3VPCEndpoint.addToPolicy( - new iam.PolicyStatement({ - principals: [new iam.AnyPrincipal()], - resources: [ - snapshotsBucket.bucketArn, - snapshotsBucket.arnForObjects("*"), - `arn:aws:s3:::al2023-repos-${region}-*`, - `arn:aws:s3:::al2023-repos-${region}-*/*`, - `arn:aws:s3:::amazonlinux-2-repos-${region}`, - `arn:aws:s3:::amazonlinux-2-repos-${region}/*`, - `arn:aws:s3:::${asset.s3BucketName}`, - `arn:aws:s3:::${asset.s3BucketName}/*`, - "arn:aws:s3:::cloudformation-examples", - "arn:aws:s3:::cloudformation-examples/*", - "arn:aws:s3:::amazoncloudwatch-agent", - "arn:aws:s3:::amazoncloudwatch-agent/*" - ], - actions: ["s3:ListBucket", "s3:*Object", "s3:GetBucket*"], - }) - ); - - s3VPCEndpoint.addToPolicy( - new iam.PolicyStatement({ - principals: [new iam.AnyPrincipal()], - resources: ["arn:aws:s3:::docker-images-prod", "arn:aws:s3:::docker-images-prod/*"], - actions: ["*"], - sid: "Allow access to docker images", - }) - ); + // If we manage snapshots ourselves with S3, create bucket, S3 endpoint, and add access rights + if (props.snapshotType === "s3") { + const snapshotsBucket = new SnapshotsS3BucketConstruct(this, `snapshots-s3-bucket`, { + bucketName: `${this.AWS_STACKNAME}-${this.AWS_ACCOUNT_ID}-${this.AWS_REGION}`, + }); + + instanceRole.addToPolicy( + new iam.PolicyStatement({ + resources: [snapshotsBucket.bucketArn, snapshotsBucket.arnForObjects("*")], + actions: ["s3:ListBucket", "s3:*Object"], + }) + ); + + const s3VPCEndpoint = vpc.addGatewayEndpoint("s3-vpc-endpoint", { + service: ec2.GatewayVpcEndpointAwsService.S3, + }); + + // Limiting access through this VPC endpoint only to our sync bucket and Amazon linux repo bucket + s3VPCEndpoint.addToPolicy( + new iam.PolicyStatement({ + principals: [new iam.AnyPrincipal()], + resources: [ + snapshotsBucket.bucketArn, + snapshotsBucket.arnForObjects("*"), + `arn:aws:s3:::al2023-repos-${region}-*`, + `arn:aws:s3:::al2023-repos-${region}-*/*`, + `arn:aws:s3:::amazonlinux-2-repos-${region}`, + `arn:aws:s3:::amazonlinux-2-repos-${region}/*`, + `arn:aws:s3:::${asset.s3BucketName}`, + `arn:aws:s3:::${asset.s3BucketName}/*`, + "arn:aws:s3:::cloudformation-examples", + "arn:aws:s3:::cloudformation-examples/*", + "arn:aws:s3:::amazoncloudwatch-agent", + "arn:aws:s3:::amazoncloudwatch-agent/*" + ], + actions: ["s3:ListBucket", "s3:*Object", "s3:GetBucket*"], + }) + ); + + s3VPCEndpoint.addToPolicy( + new iam.PolicyStatement({ + principals: [new iam.AnyPrincipal()], + resources: ["arn:aws:s3:::docker-images-prod", "arn:aws:s3:::docker-images-prod/*"], + actions: ["*"], + sid: "Allow access to docker images", + }) + ); + + new cdk.CfnOutput(this, "Snapshot Bucket Name", { + value: snapshotsBucket.bucketName, + exportName: "NodeSnapshotBucketName", + }); + } new cdk.CfnOutput(this, "Instance Role ARN", { value: instanceRole.roleArn, exportName: "NodeInstanceRoleArn", }); - new cdk.CfnOutput(this, "Snapshot Bucket Name", { - value: snapshotsBucket.bucketName, - exportName: "NodeSnapshotBucketName", - }); - /** * cdk-nag suppressions */ diff --git a/lib/ethereum/lib/config/ethConfig.interface.ts b/lib/ethereum/lib/config/node-config.interface.ts similarity index 60% rename from lib/ethereum/lib/config/ethConfig.interface.ts rename to lib/ethereum/lib/config/node-config.interface.ts index 6d2df4e0..9ea7bd1d 100644 --- a/lib/ethereum/lib/config/ethConfig.interface.ts +++ b/lib/ethereum/lib/config/node-config.interface.ts @@ -1,14 +1,23 @@ import * as configTypes from "../../../constructs/config.interface"; -export type EthClientCombination = "besu-teku" | "geth-lighthouse" | "erigon-lighthouse" | "erigon-prysm" | "nethermind-teku"; +export type EthClientCombination = "besu-teku" | "geth-lighthouse" | "erigon-lighthouse" | "erigon-prysm" | "nethermind-teku" | "reth-lighthouse"; export type EthNodeRole = "sync-node" | "rpc-node" | "single-node"; +export type EthNetwork = "mainnet" | "sepolia" | "holesky"; + +export type SnapshotType = "s3" | "none"; + export interface EthDataVolumeConfig extends configTypes.DataVolumeConfig { } export interface EthBaseConfig extends configTypes.BaseConfig { clientCombination: EthClientCombination; + network: EthNetwork + consensusCheckpointSyncURL: string; + snapshotType: SnapshotType; + consensusSnapshotURL: string; + executionSnapshotURL: string; } export interface EthSyncNodeConfig extends configTypes.SingleNodeConfig { diff --git a/lib/ethereum/lib/config/ethConfig.ts b/lib/ethereum/lib/config/node-config.ts similarity index 80% rename from lib/ethereum/lib/config/ethConfig.ts rename to lib/ethereum/lib/config/node-config.ts index 5f7cd478..d6f5dbaf 100644 --- a/lib/ethereum/lib/config/ethConfig.ts +++ b/lib/ethereum/lib/config/node-config.ts @@ -1,5 +1,5 @@ import * as ec2 from "aws-cdk-lib/aws-ec2"; -import * as configTypes from "./ethConfig.interface"; +import * as configTypes from "./node-config.interface"; import * as constants from "../../../constructs/constants"; @@ -20,8 +20,13 @@ const parseDataVolumeType = (dataVolumeType: string) => { export const baseConfig: configTypes.EthBaseConfig = { accountId: process.env.AWS_ACCOUNT_ID || "xxxxxxxxxxx", // Set your target AWS Account ID - region: process.env.AWS_REGION || "us-east-2", // Set your target AWS Region + region: process.env.AWS_REGION || "us-east-1", // Set your target AWS Region clientCombination: process.env.ETH_CLIENT_COMBINATION || "geth-lighthouse", // Set the pair of EL-CL clients : "geth-lighthouse", "erigon-lighthouse", "nethermind-teku", "besu-teku" + consensusCheckpointSyncURL: process.env.ETH_CONSENSUS_CHECKPOINT_SYNC_URL || "https://beaconstate.info", // Set the URL for the consensus checkpoint sync. Default: "https://beaconstate.info" + network: process.env.ETH_NETWORK || "mainnet", // Set the network to be used. Default: "mainnet" + snapshotType: process.env.ETH_SNAPSHOT_TYPE || constants.NoneValue, // Set the snapshot type to be used. Default: "none" + consensusSnapshotURL: process.env.ETH_CONSENSUS_SNAPSHOT_URL || constants.NoneValue, // Set the URL for the consensus snapshot. Default: "none" + executionSnapshotURL: process.env.ETH_EXECUTION_SNAPSHOT_URL || constants.NoneValue // Set the URL for the execution snapshot. Default: "none" }; export const syncNodeConfig: configTypes.EthSyncNodeConfig = { diff --git a/lib/ethereum/lib/constructs/eth-node-security-group.ts b/lib/ethereum/lib/constructs/eth-node-security-group.ts index 24abbee8..1de1d52e 100644 --- a/lib/ethereum/lib/constructs/eth-node-security-group.ts +++ b/lib/ethereum/lib/constructs/eth-node-security-group.ts @@ -1,7 +1,7 @@ import * as cdk from "aws-cdk-lib"; import * as cdkContructs from 'constructs'; import * as ec2 from "aws-cdk-lib/aws-ec2"; -import * as configTypes from "../config/ethConfig.interface" +import * as configTypes from "../config/node-config.interface" import * as nag from "cdk-nag"; export interface EthNodeSecurityGroupConstructProps { @@ -29,13 +29,14 @@ export interface EthNodeSecurityGroupConstructProps { // Public ports sg.addIngressRule(ec2.Peer.anyIpv4(), ec2.Port.tcp(30303), "P2P"); sg.addIngressRule(ec2.Peer.anyIpv4(), ec2.Port.udp(30303), "P2P"); - sg.addIngressRule(ec2.Peer.anyIpv4(), ec2.Port.tcp(30304), "P2P"); - sg.addIngressRule(ec2.Peer.anyIpv4(), ec2.Port.udp(30304), "P2P"); - sg.addIngressRule(ec2.Peer.anyIpv4(), ec2.Port.tcp(9000), "CL Client P2P"); - sg.addIngressRule(ec2.Peer.anyIpv4(), ec2.Port.udp(9000), "CL Client P2P"); if (clientCombination.startsWith("erigon")){ sg.addIngressRule(ec2.Peer.anyIpv4(), ec2.Port.tcp(42069), "Erigon Snap sync (Bittorrent)"); sg.addIngressRule(ec2.Peer.anyIpv4(), ec2.Port.udp(42069), "Erigon Snap sync (Bittorrent)"); + sg.addIngressRule(ec2.Peer.anyIpv4(), ec2.Port.udp(9000), "CL Client P2P"); + sg.addIngressRule(ec2.Peer.anyIpv4(), ec2.Port.tcp(9001), "CL Client P2P"); + } else { + sg.addIngressRule(ec2.Peer.anyIpv4(), ec2.Port.tcp(9000), "CL Client P2P"); + sg.addIngressRule(ec2.Peer.anyIpv4(), ec2.Port.udp(9000), "CL Client P2P"); } // Private ports restricted only to the VPC IP range diff --git a/lib/ethereum/lib/assets/node-cw-dashboard.ts b/lib/ethereum/lib/constructs/node-cw-dashboard.ts similarity index 99% rename from lib/ethereum/lib/assets/node-cw-dashboard.ts rename to lib/ethereum/lib/constructs/node-cw-dashboard.ts index 0ada8429..9c494c6e 100644 --- a/lib/ethereum/lib/assets/node-cw-dashboard.ts +++ b/lib/ethereum/lib/constructs/node-cw-dashboard.ts @@ -1,4 +1,4 @@ -export const NodeCWDashboardJSON = { +export const SingleNodeCWDashboardJSON = { "widgets": [ { "height": 5, diff --git a/lib/ethereum/lib/rpc-nodes-stack.ts b/lib/ethereum/lib/rpc-nodes-stack.ts index 522f71c8..f927d581 100644 --- a/lib/ethereum/lib/rpc-nodes-stack.ts +++ b/lib/ethereum/lib/rpc-nodes-stack.ts @@ -6,16 +6,22 @@ import * as s3Assets from "aws-cdk-lib/aws-s3-assets"; import * as nag from "cdk-nag"; import * as path from "path"; import * as fs from "fs"; -import * as configTypes from "./config/ethConfig.interface"; +import * as configTypes from "./config/node-config.interface"; +import * as constants from "../../constructs/constants"; import { EthNodeSecurityGroupConstruct } from "./constructs/eth-node-security-group" import { HANodesConstruct } from "../../constructs/ha-rpc-nodes-with-alb" export interface EthRpcNodesStackProps extends cdk.StackProps { ethClientCombination: configTypes.EthClientCombination; + network: configTypes.EthNetwork; + snapshotType: configTypes.SnapshotType; + consensusSnapshotURL: string; + executionSnapshotURL: string; + consensusCheckpointSyncURL: string; nodeRole: configTypes.EthNodeRole; instanceType: ec2.InstanceType; instanceCpuType: ec2.AmazonLinuxCpuType; - dataVolumes: configTypes.EthDataVolumeConfig[], + dataVolume: configTypes.EthDataVolumeConfig, numberOfNodes: number; albHealthCheckGracePeriodMin: number; heartBeatDelayMin: number; @@ -35,9 +41,14 @@ export class EthRpcNodesStack extends cdk.Stack { const { instanceType, ethClientCombination, + network, + snapshotType, + consensusSnapshotURL, + executionSnapshotURL, + consensusCheckpointSyncURL, nodeRole, instanceCpuType, - dataVolumes, + dataVolume, albHealthCheckGracePeriodMin, heartBeatDelayMin, numberOfNodes, @@ -59,20 +70,31 @@ export class EthRpcNodesStack extends cdk.Stack { // Getting the snapshot bucket name and IAM role ARN from the common stack const importedInstanceRoleArn = cdk.Fn.importValue("NodeInstanceRoleArn"); - const snapshotBucketName = cdk.Fn.importValue("NodeSnapshotBucketName"); + let snapshotBucketName; + + if (snapshotType === "s3") { + snapshotBucketName = cdk.Fn.importValue("NodeSnapshotBucketName"); + } const instanceRole = iam.Role.fromRoleArn(this, "iam-role", importedInstanceRoleArn); // Parsing user data script and injecting necessary variables - const userData = fs.readFileSync(path.join(__dirname, "assets", "user-data", "node.sh")).toString(); + const userData = fs.readFileSync(path.join(__dirname, "assets", "user-data-alinux.sh")).toString(); const modifiedUserData = cdk.Fn.sub(userData, { _REGION_: REGION, _ASSETS_S3_PATH_: `s3://${asset.s3BucketName}/${asset.s3ObjectKey}`, - _SNAPSHOT_S3_PATH_: `s3://${snapshotBucketName}/${ethClientCombination}`, + _SNAPSHOT_S3_PATH_: snapshotBucketName ? `s3://${snapshotBucketName}/${ethClientCombination}` : constants.NoneValue, _ETH_CLIENT_COMBINATION_: ethClientCombination, + _ETH_NETWORK_: network, + _ETH_SNAPSHOT_TYPE_: snapshotType, + _ETH_CONSENSUS_SNAPSHOT_URL_: consensusSnapshotURL, + _ETH_EXECUTION_SNAPSHOT_URL_: executionSnapshotURL, + _ETH_CONSENSUS_CHECKPOINT_SYNC_URL_: consensusCheckpointSyncURL, _STACK_NAME_: STACK_NAME, _FORMAT_DISK_: "true", + _DATA_VOLUME_TYPE_: dataVolume.type, + _DATA_VOLUME_SIZE_: dataVolume.sizeGiB.toString(), _NODE_ROLE_: nodeRole, _AUTOSTART_CONTAINER_: "true", _NODE_CF_LOGICAL_ID_: "", @@ -83,12 +105,12 @@ export class EthRpcNodesStack extends cdk.Stack { // Setting up the nodse using generic High Availability (HA) Node constract const rpcNodes = new HANodesConstruct (this, "rpc-nodes", { instanceType, - dataVolumes, + dataVolumes: [dataVolume], machineImage: new ec2.AmazonLinuxImage({ - generation: ec2.AmazonLinuxGeneration.AMAZON_LINUX_2, - cpuType: instanceCpuType, - kernel: ec2.AmazonLinuxKernel.KERNEL5_X, - }), + generation: ec2.AmazonLinuxGeneration.AMAZON_LINUX_2023, + kernel:ec2.AmazonLinuxKernel.KERNEL6_1, + cpuType: instanceCpuType, + }), role: instanceRole, vpc, securityGroup: instanceSG.securityGroup, diff --git a/lib/ethereum/lib/single-node-stack.ts b/lib/ethereum/lib/single-node-stack.ts index 6afaebef..1217d41d 100644 --- a/lib/ethereum/lib/single-node-stack.ts +++ b/lib/ethereum/lib/single-node-stack.ts @@ -5,19 +5,25 @@ import * as iam from "aws-cdk-lib/aws-iam"; import * as s3Assets from "aws-cdk-lib/aws-s3-assets"; import * as path from "path"; import * as fs from "fs"; -import * as nodeCwDashboard from "./assets/node-cw-dashboard" +import * as nodeCwDashboard from "./constructs/node-cw-dashboard" import * as cw from 'aws-cdk-lib/aws-cloudwatch'; import { SingleNodeConstruct } from "../../constructs/single-node" -import * as configTypes from "./config/ethConfig.interface"; +import * as configTypes from "./config/node-config.interface"; +import * as constants from "../../constructs/constants"; import { EthNodeSecurityGroupConstruct } from "./constructs/eth-node-security-group" import * as nag from "cdk-nag"; export interface EthSingleNodeStackProps extends cdk.StackProps { ethClientCombination: configTypes.EthClientCombination; + network: configTypes.EthNetwork; + snapshotType: configTypes.SnapshotType; + consensusSnapshotURL: string; + executionSnapshotURL: string; + consensusCheckpointSyncURL: string; nodeRole: configTypes.EthNodeRole; instanceType: ec2.InstanceType; instanceCpuType: ec2.AmazonLinuxCpuType; - dataVolumes: configTypes.EthDataVolumeConfig[]; + dataVolume: configTypes.EthDataVolumeConfig; } export class EthSingleNodeStack extends cdk.Stack { @@ -34,9 +40,14 @@ export class EthSingleNodeStack extends cdk.Stack { const { instanceType, ethClientCombination, + network, + snapshotType, + consensusSnapshotURL, + executionSnapshotURL, + consensusCheckpointSyncURL, nodeRole, instanceCpuType, - dataVolumes, + dataVolume, } = props; // Using default VPC @@ -55,7 +66,11 @@ export class EthSingleNodeStack extends cdk.Stack { // Getting the snapshot bucket name and IAM role ARN from the common stack const importedInstanceRoleArn = cdk.Fn.importValue("NodeInstanceRoleArn"); - const snapshotBucketName = cdk.Fn.importValue("NodeSnapshotBucketName"); + let snapshotBucketName; + + if (snapshotType === "s3") { + snapshotBucketName = cdk.Fn.importValue("NodeSnapshotBucketName"); + } const instanceRole = iam.Role.fromRoleArn(this, "iam-role", importedInstanceRoleArn); @@ -66,11 +81,11 @@ export class EthSingleNodeStack extends cdk.Stack { const node = new SingleNodeConstruct(this, "single-node", { instanceName: STACK_NAME, instanceType, - dataVolumes, - machineImage: new ec2.AmazonLinuxImage({ - generation: ec2.AmazonLinuxGeneration.AMAZON_LINUX_2, + dataVolumes: [dataVolume], + machineImage: new ec2.AmazonLinuxImage({ + generation: ec2.AmazonLinuxGeneration.AMAZON_LINUX_2023, + kernel:ec2.AmazonLinuxKernel.KERNEL6_1, cpuType: instanceCpuType, - kernel: ec2.AmazonLinuxKernel.KERNEL5_X, }), vpc, availabilityZone: chosenAvailabilityZone, @@ -82,27 +97,34 @@ export class EthSingleNodeStack extends cdk.Stack { }); // Parsing user data script and injecting necessary variables - const userData = fs.readFileSync(path.join(__dirname, "assets", "user-data", "node.sh")).toString(); + const userData = fs.readFileSync(path.join(__dirname, "assets", "user-data-alinux.sh")).toString(); const modifiedUserData = cdk.Fn.sub(userData, { _REGION_: REGION, _ASSETS_S3_PATH_: `s3://${asset.s3BucketName}/${asset.s3ObjectKey}`, - _SNAPSHOT_S3_PATH_: `s3://${snapshotBucketName}/${ethClientCombination}`, + _SNAPSHOT_S3_PATH_: snapshotBucketName ? `s3://${snapshotBucketName}/${ethClientCombination}` : constants.NoneValue, _ETH_CLIENT_COMBINATION_: ethClientCombination, + _ETH_NETWORK_: network, + _ETH_SNAPSHOT_TYPE_: snapshotType, + _ETH_CONSENSUS_SNAPSHOT_URL_: consensusSnapshotURL, + _ETH_EXECUTION_SNAPSHOT_URL_: executionSnapshotURL, + _ETH_CONSENSUS_CHECKPOINT_SYNC_URL_: consensusCheckpointSyncURL, _STACK_NAME_: STACK_NAME, _AUTOSTART_CONTAINER_: "true", _FORMAT_DISK_: "true", + _DATA_VOLUME_TYPE_: dataVolume.type, + _DATA_VOLUME_SIZE_: dataVolume.sizeGiB.toString(), _NODE_ROLE_:nodeRole, _NODE_CF_LOGICAL_ID_: node.nodeCFLogicalId, - _LIFECYCLE_HOOK_NAME_: "", - _AUTOSCALING_GROUP_NAME_: "", + _LIFECYCLE_HOOK_NAME_: constants.NoneValue, + _AUTOSCALING_GROUP_NAME_: constants.NoneValue, }); // Adding modified userdata script to the instance prepared fro us by Single Node constract node.instance.addUserData(modifiedUserData); // Adding CloudWatch dashboard to the node - const dashboardString = cdk.Fn.sub(JSON.stringify(nodeCwDashboard.NodeCWDashboardJSON), { + const dashboardString = cdk.Fn.sub(JSON.stringify(nodeCwDashboard.SingleNodeCWDashboardJSON), { INSTANCE_ID:node.instanceId, INSTANCE_NAME: STACK_NAME, REGION: REGION, diff --git a/lib/ethereum/sample-configs/.env-erigon-lighthouse b/lib/ethereum/sample-configs/.env-erigon-lighthouse index f962926e..8d30e33e 100644 --- a/lib/ethereum/sample-configs/.env-erigon-lighthouse +++ b/lib/ethereum/sample-configs/.env-erigon-lighthouse @@ -7,23 +7,34 @@ AWS_ACCOUNT_ID="xxxxxxxxxxx" AWS_REGION="us-east-2" # Common configuration parameters -ETH_CLIENT_COMBINATION="erigon-lighthouse" # All options: "besu-teku" | "geth-lighthouse" | "erigon-lighthouse" | "erigon-prysm" | "nethermind-teku" +ETH_CLIENT_COMBINATION="erigon-lighthouse" # All options: "besu-teku" | "geth-lighthouse" | "erigon-lighthouse" | "erigon-prysm" | "nethermind-teku" | "reth-lighthouse +ETH_NETWORK="mainnet" # All options: "mainnet", "sepolia", "holesky" -# Sync node configuration -ETH_SYNC_INSTANCE_TYPE="m7g.2xlarge" +# Consensus checkpoint configuration. More options: https://eth-clients.github.io/checkpoint-sync-endpoints/ +ETH_CONSENSUS_CHECKPOINT_SYNC_URL="https://beaconstate.info" # For "mainnet" +# ETH_CONSENSUS_CHECKPOINT_SYNC_URL="https://sepolia.beaconstate.info" # For "sepolia" +# ETH_CONSENSUS_CHECKPOINT_SYNC_URL="https://holesky.beaconstate.info" # For "holesky" + +# Configuration for node data snapshot management +ETH_SNAPSHOT_TYPE="none" # All options: "s3" | "none" + +# Sync node configuration (in case you used ETH_SNAPSHOT_TYPE="s3") +ETH_SYNC_INSTANCE_TYPE="r7g.2xlarge" ETH_SYNC_CPU_TYPE="ARM_64" # IMPORTANT: Make sure the CPU type matches the instance type used -ETH_SYNC_DATA_VOL_SIZE="3072" # Minimum values in Gibibytes: nethermind-teku: 2048, geth-lighthouse: 2048, besu-teku: 2048, erigon-lighthouse: 3072 +ETH_SYNC_DATA_VOL_SIZE="3072" # Minimum values in Gibibytes: nethermind-teku: 2048, geth-lighthouse: 2048, besu-teku: 2048, erigon-lighthouse: 3072, reth-lighthouse: 3072 ETH_SYNC_DATA_VOL_TYPE="gp3" # Other options: "io1" | "io2" | "gp3" | "instance-store" . IMPORTANT: Use "instance-store" option only with instance types that support that feature, like popular for node im4gn, d3, i3en, and i4i instance families ETH_SYNC_DATA_VOL_IOPS="7000" # Max IOPS for EBS volumes (not applicable for "instance-store") ETH_SYNC_DATA_VOL_THROUGHPUT="500" # Max throughput for EBS gp3 volumes (not applicable for "io1" | "io2" | "instance-store") # RPC nodes configuration -ETH_RPC_INSTANCE_TYPE="m7g.2xlarge" +ETH_RPC_INSTANCE_TYPE="i8g.4xlarge" ETH_RPC_CPU_TYPE="ARM_64" # IMPORTANT: Make sure the CPU type matches the instance type used +ETH_RPC_DATA_VOL_TYPE="instance-store" # Other options: "io1" | "io2" | "gp3" | "instance-store" . IMPORTANT: Use "instance-store" option only with instance types that support that feature, like popular for node i8g, im4gn, d3, i3en, and i4i instance families +# ETH_RPC_DATA_VOL_SIZE="3072" # Minimum values in Gibibytes: nethermind-teku: 2048, geth-lighthouse: 2048, besu-teku: 2048, erigon-lighthouse: 3072, reth-lighthouse: 3072 +# ETH_RPC_DATA_VOL_IOPS="10000" # Max IOPS for EBS volumes (not applicable for "instance-store") +# ETH_RPC_DATA_VOL_THROUGHPUT="700" # Max throughput for EBS gp3 volumes (not applicable for "io1" | "io2" | "instance-store") + +# Additional configuration for HA RPC Nodes ETH_RPC_NUMBER_OF_NODES="2" # Total number of RPC nodes to be provisioned. Default: 2 ETH_RPC_ALB_HEALTHCHECK_GRACE_PERIOD_MIN="10" # Time enough to initialize the instance and start downloading snpashots -ETH_RPC_HA_NODES_HEARTBEAT_DELAY_MIN="150" # Time sufficient enough for a node do download snapshot, from S3 bucket, start clients, and finish sycning the delta -ETH_RPC_DATA_VOL_SIZE="3072" # Minimum values in Gibibytes: nethermind-teku: 2048, geth-lighthouse: 2048, besu-teku: 2048, erigon-lighthouse: 3072 -ETH_RPC_DATA_VOL_TYPE="gp3" # Other options: "io1" | "io2" | "gp3" | "instance-store" . IMPORTANT: Use "instance-store" option only with instance types that support that feature, like popular for node im4gn, d3, i3en, and i4i instance families -ETH_RPC_DATA_VOL_IOPS="10000" # Max IOPS for EBS volumes (not applicable for "instance-store") -ETH_RPC_DATA_VOL_THROUGHPUT="700" # Max throughput for EBS gp3 volumes (not applicable for "io1" | "io2" | "instance-store") +ETH_RPC_HA_NODES_HEARTBEAT_DELAY_MIN="60" # Time sufficient enough for a node do download snapshot, from S3 bucket, start clients, and finish sycning the delta diff --git a/lib/ethereum/sample-configs/.env-erigon-prysm b/lib/ethereum/sample-configs/.env-erigon-prysm deleted file mode 100644 index bea87fe0..00000000 --- a/lib/ethereum/sample-configs/.env-erigon-prysm +++ /dev/null @@ -1,29 +0,0 @@ -############################################################## -# Example configuration for Ethereum nodes runner app on AWS # -############################################################## - -# Set the AWS account is and region for your environment -AWS_ACCOUNT_ID="xxxxxxxxxxx" -AWS_REGION="us-east-2" - -# Common configuration parameters -ETH_CLIENT_COMBINATION="erigon-prysm" # All options: "besu-teku" | "geth-lighthouse" | "erigon-lighthouse" | "erigon-prysm" | "nethermind-teku" - -# Sync node configuration -ETH_SYNC_INSTANCE_TYPE="m7g.2xlarge" -ETH_SYNC_CPU_TYPE="ARM_64" # IMPORTANT: Make sure the CPU type matches the instance type used -ETH_SYNC_DATA_VOL_SIZE="3072" # Minimum values in Gibibytes: nethermind-teku: 2048, geth-lighthouse: 2048, besu-teku: 2048, erigon-lighthouse: 3072 -ETH_SYNC_DATA_VOL_TYPE="gp3" # Other options: "io1" | "io2" | "gp3" | "instance-store" . IMPORTANT: Use "instance-store" option only with instance types that support that feature, like popular for node im4gn, d3, i3en, and i4i instance families -ETH_SYNC_DATA_VOL_IOPS="7000" # Max IOPS for EBS volumes (not applicable for "instance-store") -ETH_SYNC_DATA_VOL_THROUGHPUT="500" # Max throughput for EBS gp3 volumes (not applicable for "io1" | "io2" | "instance-store") - -# RPC nodes configuration -ETH_RPC_INSTANCE_TYPE="m7g.2xlarge" -ETH_RPC_CPU_TYPE="ARM_64" # IMPORTANT: Make sure the CPU type matches the instance type used -ETH_RPC_NUMBER_OF_NODES="2" # Total number of RPC nodes to be provisioned. Default: 2 -ETH_RPC_ALB_HEALTHCHECK_GRACE_PERIOD_MIN="10" # Time enough to initialize the instance and start downloading snpashots -ETH_RPC_HA_NODES_HEARTBEAT_DELAY_MIN="150" # Time sufficient enough for a node do download snapshot, from S3 bucket, start clients, and finish sycning the delta -ETH_RPC_DATA_VOL_SIZE="3072" # Minimum values in Gibibytes: nethermind-teku: 2048, geth-lighthouse: 2048, besu-teku: 2048, erigon-lighthouse: 3072 -ETH_RPC_DATA_VOL_TYPE="gp3" # Other options: "io1" | "io2" | "gp3" | "instance-store" . IMPORTANT: Use "instance-store" option only with instance types that support that feature, like popular for node im4gn, d3, i3en, and i4i instance families -ETH_RPC_DATA_VOL_IOPS="10000" # Max IOPS for EBS volumes (not applicable for "instance-store") -ETH_RPC_DATA_VOL_THROUGHPUT="700" # Max throughput for EBS gp3 volumes (not applicable for "io1" | "io2" | "instance-store") diff --git a/lib/ethereum/sample-configs/.env-geth-lighthouse b/lib/ethereum/sample-configs/.env-geth-lighthouse index f9971787..b6ad4bcb 100644 --- a/lib/ethereum/sample-configs/.env-geth-lighthouse +++ b/lib/ethereum/sample-configs/.env-geth-lighthouse @@ -4,26 +4,37 @@ # Set the AWS account is and region for your environment AWS_ACCOUNT_ID="xxxxxxxxxxx" -AWS_REGION="us-east-2" +AWS_REGION="us-east-1" # Common configuration parameters -ETH_CLIENT_COMBINATION="geth-lighthouse" # All options: "besu-teku" | "geth-lighthouse" | "erigon-lighthouse" | "erigon-prysm" | "nethermind-teku" +ETH_CLIENT_COMBINATION="geth-lighthouse" # All options: "besu-teku" | "geth-lighthouse" | "erigon-lighthouse" | "erigon-prysm" | "nethermind-teku" | "reth-lighthouse +ETH_NETWORK="mainnet" # All options: "mainnet", "sepolia", "holesky" -# Sync node configuration -ETH_SYNC_INSTANCE_TYPE="m6g.2xlarge" +# Consensus checkpoint configuration. More options: https://eth-clients.github.io/checkpoint-sync-endpoints/ +ETH_CONSENSUS_CHECKPOINT_SYNC_URL="https://beaconstate.info" # For "mainnet" +# ETH_CONSENSUS_CHECKPOINT_SYNC_URL="https://sepolia.beaconstate.info" # For "sepolia" +# ETH_CONSENSUS_CHECKPOINT_SYNC_URL="https://holesky.beaconstate.info" # For "holesky" + +# Configuration for node data snapshot management +ETH_SNAPSHOT_TYPE="none" # All options: "s3" | "none" + +# Sync node configuration (in case you used ETH_SNAPSHOT_TYPE="s3") +ETH_SYNC_INSTANCE_TYPE="r7g.2xlarge" ETH_SYNC_CPU_TYPE="ARM_64" # IMPORTANT: Make sure the CPU type matches the instance type used -ETH_SYNC_DATA_VOL_SIZE="3072" # Minimum values in Gibibytes: nethermind-teku: 2048, geth-lighthouse: 2048, besu-teku: 2048, erigon-lighthouse: 3072 +ETH_SYNC_DATA_VOL_SIZE="3072" # Minimum values in Gibibytes: nethermind-teku: 2048, geth-lighthouse: 2048, besu-teku: 2048, erigon-lighthouse: 3072, reth-lighthouse: 3072 ETH_SYNC_DATA_VOL_TYPE="gp3" # Other options: "io1" | "io2" | "gp3" | "instance-store" . IMPORTANT: Use "instance-store" option only with instance types that support that feature, like popular for node im4gn, d3, i3en, and i4i instance families ETH_SYNC_DATA_VOL_IOPS="6000" # Max IOPS for EBS volumes (not applicable for "instance-store") ETH_SYNC_DATA_VOL_THROUGHPUT="400" # Max throughput for EBS gp3 volumes (not applicable for "io1" | "io2" | "instance-store") # RPC nodes configuration -ETH_RPC_INSTANCE_TYPE="m7g.2xlarge" +ETH_RPC_INSTANCE_TYPE="r7g.2xlarge" ETH_RPC_CPU_TYPE="ARM_64" # IMPORTANT: Make sure the CPU type matches the instance type used -ETH_RPC_NUMBER_OF_NODES="2" # Total number of RPC nodes to be provisioned. Default: 2 -ETH_RPC_ALB_HEALTHCHECK_GRACE_PERIOD_MIN="10" # Time enough to initialize the instance and start downloading snpashots -ETH_RPC_HA_NODES_HEARTBEAT_DELAY_MIN="60" # Time sufficient enough for a node do download snapshot, from S3 bucket, start clients, and finish sycning the delta -ETH_RPC_DATA_VOL_SIZE="3072" # Minimum values in Gibibytes: nethermind-teku: 2048, geth-lighthouse: 2048, besu-teku: 2048, erigon-lighthouse: 3072 +ETH_RPC_DATA_VOL_SIZE="3072" # Minimum values in Gibibytes: nethermind-teku: 2048, geth-lighthouse: 2048, besu-teku: 2048, erigon-lighthouse: 3072, reth-lighthouse: 3072 ETH_RPC_DATA_VOL_TYPE="gp3" # Other options: "io1" | "io2" | "gp3" | "instance-store" . IMPORTANT: Use "instance-store" option only with instance types that support that feature, like popular for node im4gn, d3, i3en, and i4i instance families ETH_RPC_DATA_VOL_IOPS="8000" # Max IOPS for EBS volumes (not applicable for "instance-store") ETH_RPC_DATA_VOL_THROUGHPUT="700" # Max throughput for EBS gp3 volumes (not applicable for "io1" | "io2" | "instance-store") + +# Additional configuration for HA RPC Nodes +ETH_RPC_NUMBER_OF_NODES="2" # Total number of RPC nodes to be provisioned. Default: 2 +ETH_RPC_ALB_HEALTHCHECK_GRACE_PERIOD_MIN="10" # Time enough to initialize the instance and start downloading snpashots +ETH_RPC_HA_NODES_HEARTBEAT_DELAY_MIN="60" # Time sufficient enough for a node do download snapshot, from S3 bucket, start clients, and finish sycning the delta diff --git a/lib/ethereum/sample-configs/.env-reth-lighthouse b/lib/ethereum/sample-configs/.env-reth-lighthouse new file mode 100644 index 00000000..d212eb81 --- /dev/null +++ b/lib/ethereum/sample-configs/.env-reth-lighthouse @@ -0,0 +1,40 @@ +############################################################## +# Example configuration for Ethereum nodes runner app on AWS # +############################################################## + +# Set the AWS account is and region for your environment +AWS_ACCOUNT_ID="xxxxxxxxxxx" +AWS_REGION="us-east-2" + +# Common configuration parameters +ETH_CLIENT_COMBINATION="reth-lighthouse" # All options: "besu-teku" | "geth-lighthouse" | "erigon-lighthouse" | "erigon-prysm" | "nethermind-teku" | "reth-lighthouse +ETH_NETWORK="mainnet" # All options: "mainnet", "sepolia", "holesky" + +# Consensus checkpoint configuration. More options: https://eth-clients.github.io/checkpoint-sync-endpoints/ +ETH_CONSENSUS_CHECKPOINT_SYNC_URL="https://beaconstate.info" # For "mainnet" +# ETH_CONSENSUS_CHECKPOINT_SYNC_URL="https://sepolia.beaconstate.info" # For "sepolia" +# ETH_CONSENSUS_CHECKPOINT_SYNC_URL="https://holesky.beaconstate.info" # For "holesky" + +# Configuration for node data snapshot management +ETH_SNAPSHOT_TYPE="none" # All options: "s3" | "none" + +# Sync node configuration (in case you used ETH_SNAPSHOT_TYPE="s3") +ETH_SYNC_INSTANCE_TYPE="r7g.2xlarge" +ETH_SYNC_CPU_TYPE="ARM_64" # IMPORTANT: Make sure the CPU type matches the instance type used +ETH_SYNC_DATA_VOL_SIZE="3072" # Minimum values in Gibibytes: nethermind-teku: 2048, geth-lighthouse: 2048, besu-teku: 2048, erigon-lighthouse: 3072, reth-lighthouse: 3072 +ETH_SYNC_DATA_VOL_TYPE="gp3" # Other options: "io1" | "io2" | "gp3" | "instance-store" . IMPORTANT: Use "instance-store" option only with instance types that support that feature, like popular for node im4gn, d3, i3en, and i4i instance families +ETH_SYNC_DATA_VOL_IOPS="10000" # Max IOPS for EBS volumes (not applicable for "instance-store") +ETH_SYNC_DATA_VOL_THROUGHPUT="500" # Max throughput for EBS gp3 volumes (not applicable for "io1" | "io2" | "instance-store") + +# RPC nodes configuration +ETH_RPC_INSTANCE_TYPE="i8g.4xlarge" +ETH_RPC_CPU_TYPE="ARM_64" # IMPORTANT: Make sure the CPU type matches the instance type used +ETH_RPC_DATA_VOL_TYPE="instance-store" # Other options: "io1" | "io2" | "gp3" | "instance-store" . IMPORTANT: Use "instance-store" option only with instance types that support that feature, like popular for node i8g, im4gn, d3, i3en, and i4i instance families +# ETH_RPC_DATA_VOL_SIZE="3072" # Minimum values in Gibibytes: nethermind-teku: 2048, geth-lighthouse: 2048, besu-teku: 2048, erigon-lighthouse: 3072, reth-lighthouse: 3072 +# ETH_RPC_DATA_VOL_IOPS="10000" # Max IOPS for EBS volumes (not applicable for "instance-store") +# ETH_RPC_DATA_VOL_THROUGHPUT="700" # Max throughput for EBS gp3 volumes (not applicable for "io1" | "io2" | "instance-store") + +# Additional configuration for HA RPC Nodes +ETH_RPC_NUMBER_OF_NODES="2" # Total number of RPC nodes to be provisioned. Default: 2 +ETH_RPC_ALB_HEALTHCHECK_GRACE_PERIOD_MIN="10" # Time enough to initialize the instance and start downloading snpashots +ETH_RPC_HA_NODES_HEARTBEAT_DELAY_MIN="60" # Time sufficient enough for a node do download snapshot, from S3 bucket, start clients, and finish sycning the delta diff --git a/lib/ethereum/test/.env-test b/lib/ethereum/test/.env-test index 694c49b2..6b107caa 100644 --- a/lib/ethereum/test/.env-test +++ b/lib/ethereum/test/.env-test @@ -8,6 +8,18 @@ AWS_REGION="us-east-2" # Common configuration parameters ETH_CLIENT_COMBINATION="geth-lighthouse" # All options: "besu-teku" | "geth-lighthouse" | "erigon-lighthouse" | "erigon-prysm" | "nethermind-teku" +ETH_NETWORK="mainnet" # All options: "mainnet", "sepolia", "holesky" + +# Consensus checkpoint configuration. More options: https://eth-clients.github.io/checkpoint-sync-endpoints/ +ETH_CONSENSUS_CHECKPOINT_SYNC_URL="https://beaconstate.info" # For "mainnet" +# ETH_CONSENSUS_CHECKPOINT_SYNC_URL="https://sepolia.beaconstate.info" # For "sepolia" +# ETH_CONSENSUS_CHECKPOINT_SYNC_URL="https://holesky.beaconstate.info" # For "holesky" + +# Configuration for node data snapshot management +ETH_SNAPSHOT_TYPE="s3" # All options: "s3" | "publicnode" +# ETH_CONSENSUS_SNAPSHOT_URL="https://snapshots.publicnode.com/ethereum-teku-10942048.tar.lz4" # Url from external source like https://publicnode.com/snapshots +# ETH_EXECUTION_SNAPSHOT_URL="https://snapshots.publicnode.com/ethereum-besu-21728106.tar.lz4" # Url from external source like https://publicnode.com/snapshots + # Sync node configuration ETH_SYNC_INSTANCE_TYPE="m6g.2xlarge" @@ -20,10 +32,12 @@ ETH_SYNC_DATA_VOL_THROUGHPUT="400" # Max throughput for EBS gp3 vol # RPC nodes configuration ETH_RPC_INSTANCE_TYPE="m7g.2xlarge" ETH_RPC_CPU_TYPE="ARM_64" # IMPORTANT: Make sure the CPU type matches the instance type used -ETH_RPC_NUMBER_OF_NODES="2" # Total number of RPC nodes to be provisioned. Default: 2 -ETH_RPC_ALB_HEALTHCHECK_GRACE_PERIOD_MIN="10" # Time enough to initialize the instance and start downloading snpashots -ETH_RPC_HA_NODES_HEARTBEAT_DELAY_MIN="60" # Time sufficient enough for a node do download snapshot, from S3 bucket, start clients, and finish sycning the delta ETH_RPC_DATA_VOL_SIZE="3072" # Minimum values in Gibibytes: nethermind-teku: 2048, geth-lighthouse: 2048, besu-teku: 2048, erigon-lighthouse: 3072 ETH_RPC_DATA_VOL_TYPE="gp3" # Other options: "io1" | "io2" | "gp3" | "instance-store" . IMPORTANT: Use "instance-store" option only with instance types that support that feature, like popular for node im4gn, d3, i3en, and i4i instance families ETH_RPC_DATA_VOL_IOPS="8000" # Max IOPS for EBS volumes (not applicable for "instance-store") ETH_RPC_DATA_VOL_THROUGHPUT="700" # Max throughput for EBS gp3 volumes (not applicable for "io1" | "io2" | "instance-store") + +# Additional configuration for HA RPC Nodes +ETH_RPC_NUMBER_OF_NODES="2" # Total number of RPC nodes to be provisioned. Default: 2 +ETH_RPC_ALB_HEALTHCHECK_GRACE_PERIOD_MIN="10" # Time enough to initialize the instance and start downloading snpashots +ETH_RPC_HA_NODES_HEARTBEAT_DELAY_MIN="60" # Time sufficient enough for a node do download snapshot, from S3 bucket, start clients, and finish sycning the delta diff --git a/lib/ethereum/test/common-stack.test.ts b/lib/ethereum/test/common-stack.test.ts index 31f57139..2409325f 100644 --- a/lib/ethereum/test/common-stack.test.ts +++ b/lib/ethereum/test/common-stack.test.ts @@ -2,7 +2,7 @@ import { Match, Template } from "aws-cdk-lib/assertions"; import * as cdk from "aws-cdk-lib"; import * as dotenv from 'dotenv'; dotenv.config({ path: './test/.env-test' }); -import * as config from "../lib/config/ethConfig"; +import * as config from "../lib/config/node-config"; import { EthCommonStack } from "../lib/common-stack"; describe("EthCommonStack", () => { @@ -13,6 +13,7 @@ describe("EthCommonStack", () => { const ethCommonStack = new EthCommonStack(app, "eth-common", { env: { account: config.baseConfig.accountId, region: config.baseConfig.region }, stackName: `eth-nodes-common`, + snapshotType: config.baseConfig.snapshotType, }); // Prepare the stack for assertions. @@ -20,7 +21,7 @@ describe("EthCommonStack", () => { // Has Snapshot S3 bucket. template.hasResourceProperties("AWS::S3::Bucket", { - BucketName: Match.stringLikeRegexp("eth-snapshots*"), + BucketName: Match.stringLikeRegexp("eth-*"), AccessControl: "Private", BucketEncryption: { ServerSideEncryptionConfiguration: [ diff --git a/lib/ethereum/test/rpc-nodes-stack.test.ts b/lib/ethereum/test/rpc-nodes-stack.test.ts index a1680046..e558d2e8 100644 --- a/lib/ethereum/test/rpc-nodes-stack.test.ts +++ b/lib/ethereum/test/rpc-nodes-stack.test.ts @@ -2,27 +2,32 @@ import { Match, Template } from "aws-cdk-lib/assertions"; import * as cdk from "aws-cdk-lib"; import * as dotenv from 'dotenv'; dotenv.config({ path: './test/.env-test' }); -import * as config from "../lib/config/ethConfig"; +import * as config from "../lib/config/node-config"; import { EthRpcNodesStack } from "../lib/rpc-nodes-stack"; -import { EthNodeRole } from "../lib/config/ethConfig.interface"; +import { EthNodeRole } from "../lib/config/node-config.interface"; describe("EthRpcNodesStack", () => { test("synthesizes the way we expect", () => { const app = new cdk.App(); // Create the EthRpcNodesStack. - const ethRpcNodesStack = new EthRpcNodesStack(app, "eth-sync-node", { + const ethRpcNodesStack = new EthRpcNodesStack(app, "eth-rpc-nodes", { stackName: `eth-rpc-nodes-${config.baseConfig.clientCombination}`, env: { account: config.baseConfig.accountId, region: config.baseConfig.region }, ethClientCombination: config.baseConfig.clientCombination, + network: config.baseConfig.network, + snapshotType: config.baseConfig.snapshotType, + consensusSnapshotURL: config.baseConfig.consensusSnapshotURL, + executionSnapshotURL: config.baseConfig.executionSnapshotURL, + consensusCheckpointSyncURL: config.baseConfig.consensusCheckpointSyncURL, nodeRole: "single-node", instanceType: config.rpcNodeConfig.instanceType, instanceCpuType: config.rpcNodeConfig.instanceCpuType, numberOfNodes: config.rpcNodeConfig.numberOfNodes, albHealthCheckGracePeriodMin: config.rpcNodeConfig.albHealthCheckGracePeriodMin, heartBeatDelayMin: config.rpcNodeConfig.heartBeatDelayMin, - dataVolumes: config.syncNodeConfig.dataVolumes, + dataVolume: config.syncNodeConfig.dataVolumes[0], }); // Prepare the stack for assertions. @@ -54,20 +59,6 @@ describe("EthRpcNodesStack", () => { "IpProtocol": "udp", "ToPort": 30303 }, - { - "CidrIp": "0.0.0.0/0", - "Description": "P2P", - "FromPort": 30304, - "IpProtocol": "tcp", - "ToPort": 30304 - }, - { - "CidrIp": "0.0.0.0/0", - "Description": "P2P", - "FromPort": 30304, - "IpProtocol": "udp", - "ToPort": 30304 - }, { "CidrIp": "0.0.0.0/0", "Description": "CL Client P2P", diff --git a/lib/ethereum/test/solo-node-stack.test.ts b/lib/ethereum/test/solo-node-stack.test.ts index 002cfa55..cc7e7789 100644 --- a/lib/ethereum/test/solo-node-stack.test.ts +++ b/lib/ethereum/test/solo-node-stack.test.ts @@ -2,9 +2,9 @@ import { Match, Template } from "aws-cdk-lib/assertions"; import * as cdk from "aws-cdk-lib"; import * as dotenv from 'dotenv'; dotenv.config({ path: './test/.env-test' }); -import * as config from "../lib/config/ethConfig"; +import * as config from "../lib/config/node-config"; import { EthSingleNodeStack } from "../lib/single-node-stack"; -import { EthNodeRole } from "../lib/config/ethConfig.interface"; +import { EthNodeRole } from "../lib/config/node-config.interface"; describe("EthSyncNodeStack", () => { test("synthesizes the way we expect", () => { @@ -16,10 +16,15 @@ describe("EthSyncNodeStack", () => { env: { account: config.baseConfig.accountId, region: config.baseConfig.region }, ethClientCombination: config.baseConfig.clientCombination, + network: config.baseConfig.network, + snapshotType: config.baseConfig.snapshotType, + consensusSnapshotURL: config.baseConfig.consensusSnapshotURL, + executionSnapshotURL: config.baseConfig.executionSnapshotURL, + consensusCheckpointSyncURL: config.baseConfig.consensusCheckpointSyncURL, nodeRole: "single-node", instanceType: config.syncNodeConfig.instanceType, instanceCpuType: config.syncNodeConfig.instanceCpuType, - dataVolumes: config.syncNodeConfig.dataVolumes, + dataVolume: config.syncNodeConfig.dataVolumes[0], }); // Prepare the stack for assertions. @@ -51,20 +56,6 @@ describe("EthSyncNodeStack", () => { "IpProtocol": "udp", "ToPort": 30303 }, - { - "CidrIp": "0.0.0.0/0", - "Description": "P2P", - "FromPort": 30304, - "IpProtocol": "tcp", - "ToPort": 30304 - }, - { - "CidrIp": "0.0.0.0/0", - "Description": "P2P", - "FromPort": 30304, - "IpProtocol": "udp", - "ToPort": 30304 - }, { "CidrIp": "0.0.0.0/0", "Description": "CL Client P2P", diff --git a/lib/ethereum/test/sync-node-stack.test.ts b/lib/ethereum/test/sync-node-stack.test.ts index 959e737b..a44c345b 100644 --- a/lib/ethereum/test/sync-node-stack.test.ts +++ b/lib/ethereum/test/sync-node-stack.test.ts @@ -2,9 +2,9 @@ import { Match, Template } from "aws-cdk-lib/assertions"; import * as cdk from "aws-cdk-lib"; import * as dotenv from 'dotenv'; dotenv.config({ path: './test/.env-test' }); -import * as config from "../lib/config/ethConfig"; +import * as config from "../lib/config/node-config"; import { EthSingleNodeStack } from "../lib/single-node-stack"; -import { EthNodeRole } from "../lib/config/ethConfig.interface"; +import { EthNodeRole } from "../lib/config/node-config.interface"; describe("EthSyncNodeStack", () => { test("synthesizes the way we expect", () => { @@ -16,10 +16,15 @@ describe("EthSyncNodeStack", () => { env: { account: config.baseConfig.accountId, region: config.baseConfig.region }, ethClientCombination: config.baseConfig.clientCombination, + network: config.baseConfig.network, + snapshotType: config.baseConfig.snapshotType, + consensusSnapshotURL: config.baseConfig.consensusSnapshotURL, + executionSnapshotURL: config.baseConfig.executionSnapshotURL, + consensusCheckpointSyncURL: config.baseConfig.consensusCheckpointSyncURL, nodeRole: "sync-node", instanceType: config.syncNodeConfig.instanceType, instanceCpuType: config.syncNodeConfig.instanceCpuType, - dataVolumes: config.syncNodeConfig.dataVolumes, + dataVolume: config.syncNodeConfig.dataVolumes[0], }); // Prepare the stack for assertions. @@ -51,20 +56,6 @@ describe("EthSyncNodeStack", () => { "IpProtocol": "udp", "ToPort": 30303 }, - { - "CidrIp": "0.0.0.0/0", - "Description": "P2P", - "FromPort": 30304, - "IpProtocol": "tcp", - "ToPort": 30304 - }, - { - "CidrIp": "0.0.0.0/0", - "Description": "P2P", - "FromPort": 30304, - "IpProtocol": "udp", - "ToPort": 30304 - }, { "CidrIp": "0.0.0.0/0", "Description": "CL Client P2P",