refactor to typescript along with several other changes to better align with other bluepritns

Author: Simon Goldberg

lib/bitcoin-core/README.md

@@ -0,0 +1,31 @@
+#!/usr/bin/env node
+import 'dotenv/config';
+import { App, Aspects } from 'aws-cdk-lib';
+import { AwsSolutionsChecks } from 'cdk-nag';
+import { BitcoinCommonStack } from './lib/common-infra';
+import { SingleNodeBitcoinCoreStack } from './lib/single-node-stack';
+import { HABitcoinCoreNodeStack } from './lib/ha-node-stack';
+import * as config from './lib/config/bitcoinConfig';
+
+const app = new App();
+
+Aspects.of(app).add(new AwsSolutionsChecks({ verbose: true }));
+
+const env = {
+  account: process.env.AWS_ACCOUNT_ID,
+  region: process.env.AWS_REGION,
+};
+
+const commonStack = new BitcoinCommonStack(app, 'BitcoinCommonStack', { env });
+new SingleNodeBitcoinCoreStack(app, 'SingleNodeBitcoinCoreStack', {
+    env,
+    instanceRole: commonStack.instanceRole,
+    ...config.baseNodeConfig,
+});
+
+new HABitcoinCoreNodeStack(app, 'HABitcoinCoreNodeStack', {
+    env,
+    instanceRole: commonStack.instanceRole,
+    ...config.baseNodeConfig,
+    ...config.haNodeConfig,
+});

lib/bitcoin-core/app.ts

@@ -0,0 +1,57 @@
+{
+  "app": "npx ts-node --prefer-ts-exts app.ts",
+  "watch": {
+    "include": [
+      "**"
+    ],
+    "exclude": [
+      "README.md",
+      "cdk*.json",
+      "**/*.d.ts",
+      "**/*.js",
+      "tsconfig.json",
+      "package*.json",
+      "yarn.lock",
+      "node_modules",
+      "test"
+    ]
+  },
+  "context": {
+    "@aws-cdk/aws-lambda:recognizeLayerVersion": true,
+    "@aws-cdk/core:checkSecretUsage": true,
+    "@aws-cdk/core:target-partitions": [
+      "aws",
+      "aws-cn"
+    ],
+    "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true,
+    "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true,
+    "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true,
+    "@aws-cdk/aws-iam:minimizePolicies": true,
+    "@aws-cdk/core:validateSnapshotRemovalPolicy": true,
+    "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true,
+    "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true,
+    "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true,
+    "@aws-cdk/aws-apigateway:disableCloudWatchRole": true,
+    "@aws-cdk/core:enablePartitionLiterals": true,
+    "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true,
+    "@aws-cdk/aws-iam:standardizedServicePrincipals": true,
+    "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true,
+    "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true,
+    "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true,
+    "@aws-cdk/aws-route53-patters:useCertificate": true,
+    "@aws-cdk/customresources:installLatestAwsSdkDefault": false,
+    "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true,
+    "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true,
+    "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true,
+    "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true,
+    "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true,
+    "@aws-cdk/aws-redshift:columnId": true,
+    "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true,
+    "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true,
+    "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true,
+    "@aws-cdk/aws-kms:aliasNameRef": true,
+    "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true,
+    "@aws-cdk/core:includePrefixInUniqueNameGeneration": true,
+    "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true
+  }
+}

lib/bitcoin-core/cdk.json

@@ -0,0 +1,8 @@
+module.exports = {
+  testEnvironment: 'node',
+  roots: ['<rootDir>/test'],
+  testMatch: ['**/*.test.ts'],
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest'
+  }
+};

lib/bitcoin-core/doc/assets/Bitcoin-HA-Nodes-Arch.png

@@ -0,0 +1,63 @@
+#!/bin/bash
+# This script is used to set up a mainnet Bitcoin Core node on an Amazon Linux 2 instance.
+set -euo pipefail
+
+# The stack passes lifecycle hook information when used in an Auto
+# Scaling Group. If no lifecycle hook name is supplied (single node
+# deployment), default to "none" and use CloudFormation signaling.
+LIFECYCLE_HOOK_NAME=${LIFECYCLE_HOOK_NAME:-none}
+AUTOSCALING_GROUP_NAME=${AUTOSCALING_GROUP_NAME:-none}
+
+# Ensure the data volume is mounted before proceeding
+until mountpoint -q /home/bitcoin; do
+  echo "Waiting for /home/bitcoin to be mounted..."
+  sleep 2
+done
+
+yum update -y
+amazon-linux-extras install docker -y
+service docker start
+systemctl enable docker
+
+# Create bitcoin user with specific UID:GID to match the container's expected values
+if id -u bitcoin > /dev/null 2>&1; then
+  # User exists, update to correct UID:GID
+  usermod -u 101 bitcoin
+  groupmod -g 101 bitcoin
+else
+  # Create user with specific UID:GID
+  groupadd -g 101 bitcoin
+  useradd -u 101 -g 101 -m -s /bin/bash bitcoin
+fi
+
+# Create the bitcoin data directory structure on the mounted EBS volume
+mkdir -p /home/bitcoin/.bitcoin
+echo "${BITCOIN_CONF}" > /home/bitcoin/.bitcoin/bitcoin.conf
+
+# Set proper permissions for the Bitcoin configuration
+chown -R bitcoin:bitcoin /home/bitcoin
+chmod -R 755 /home/bitcoin
+
+# Run Bitcoin Core in Docker with proper volume mapping
+# Modified to ensure data is stored on the EBS volume
+docker run -d --name bitcoind \
+  -v /home/bitcoin/.bitcoin:/home/bitcoin/.bitcoin \
+  -p 8333:8333 \
+  -p 8332:8332 \
+  --restart unless-stopped \
+  bitcoin/bitcoin:latest \
+  bitcoind -datadir=/home/bitcoin/.bitcoin
+
+# Signal completion depending on deployment type
+if [[ "$LIFECYCLE_HOOK_NAME" != "none" ]]; then
+  echo "Signaling ASG lifecycle hook to complete"
+  TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
+  INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/instance-id)
+  aws autoscaling complete-lifecycle-action --lifecycle-action-result CONTINUE --instance-id "$INSTANCE_ID" --lifecycle-hook-name "$LIFECYCLE_HOOK_NAME" --auto-scaling-group-name "$AUTOSCALING_GROUP_NAME" --region "$AWS_REGION"
+else
+  if ! command -v cfn-signal &> /dev/null; then
+    yum install -y aws-cfn-bootstrap
+  fi
+  cfn-signal --stack "$STACK_NAME" --resource "$RESOURCE_ID" --region "$AWS_REGION"
+fi
+

lib/bitcoin-core/doc/assets/Bitcoin-Single-Node-Arch.png

@@ -0,0 +1,4 @@
+#!/bin/bash
+# This script is used to set up a cron job to send the Bitcoin block height to Amazon CloudWatch every 5 minutes.
+REGION=${AWS_REGION}
+(crontab -l 2>/dev/null; echo "*/5 * * * * sudo /usr/bin/docker exec bitcoind bitcoin-cli getblockcount | xargs -I {} sudo /usr/bin/aws cloudwatch put-metric-data --metric-name BlockHeight --namespace Bitcoin --unit Count --value {} --region $REGION") | crontab -

lib/bitcoin-core/jest.config.js

@@ -0,0 +1,30 @@
+#!/bin/bash
+# This script is used to set up the Amazon CloudWatch agent on an Amazon Linux 2 instance.
+
+yum install -y amazon-cloudwatch-agent
+mkdir -p /opt/aws/amazon-cloudwatch-agent/etc
+cat <<EOF > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json
+{
+  "metrics": {
+    "metrics_collected": {
+      "disk": {
+        "measurement": ["used_percent", "inodes_free"],
+        "resources": ["*"],
+        "ignore_file_system_types": ["sysfs", "devtmpfs"]
+      },
+      "mem": {
+        "measurement": ["mem_used_percent"]
+      },
+      "cpu": {
+        "measurement": ["cpu_usage_idle", "cpu_usage_user", "cpu_usage_system"]
+      },
+      "net": {
+        "measurement": ["net_bytes_sent", "net_bytes_recv"],
+        "resources": ["eth0"]
+      }
+    }
+  }
+}
+EOF
+
+/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s

lib/bitcoin-core/lib/assets/bitcoin-setup.sh

@@ -0,0 +1,171 @@
+#!/bin/bash
+set -euo pipefail
+
+make_fs () {
+  # If file system = to ext4 use mkfs.ext4, if xfs use mkfs.xfs
+  if [ -z "$1" ]; then
+    echo "Error: No file system type provided."
+    echo "Usage: make_fs <file system type [ xfs | ext4 ]> <target_volume_id>"
+    exit 1
+  fi
+
+  if [ -z "$2" ]; then
+    echo "Error: No target volume ID provided."
+    echo "Usage: make_fs <file system type [ xfs | ext4 ]> <target_volume_id>"
+    exit 1
+  fi
+
+  local file_system=$1
+  local volume_id=$2
+  if [ "$file_system" == "ext4" ]; then
+    mkfs -t ext4 "$volume_id"
+    return "$?"
+  else
+    mkfs.xfs -f "$volume_id"
+    return "$?"
+  fi
+}
+
+# We need an nvme disk that is not mounted and not partitioned
+get_all_empty_nvme_disks () {
+  local all_not_mounted_nvme_disks
+  local all_mounted_nvme_partitions
+  local unmounted_nvme_disks=()
+  local sorted_unmounted_nvme_disks
+
+  #The disk will only be mounted when the nvme disk is larger than 100GB to avoid storing blockchain node data directly on the root EBS disk (which is 46GB by default)
+  all_not_mounted_nvme_disks=$(lsblk -lnb | awk '{if ($7 == "" && $4 > 107374182400) {print $1}}' | grep nvme)
+  all_mounted_nvme_partitions=$(mount | awk '{print $1}' | grep /dev/nvme)
+  for disk in ${all_not_mounted_nvme_disks[*]}; do
+    if [[ ! "${all_mounted_nvme_partitions[*]}" =~ $disk ]]; then
+      unmounted_nvme_disks+=("$disk")
+    fi
+  done
+  # Sort the array
+  sorted_unmounted_nvme_disks=($(printf '%s\n' "${unmounted_nvme_disks[*]}" | sort))
+  echo "${sorted_unmounted_nvme_disks[*]}"
+}
+
+get_next_empty_nvme_disk () {
+  local sorted_unmounted_nvme_disks
+  sorted_unmounted_nvme_disks=($(get_all_empty_nvme_disks))
+  # Return the first unmounted nvme disk
+  echo "/dev/${sorted_unmounted_nvme_disks[0]}"
+}
+
+# Add input as command line parameters for name of the directory to mount
+if [ -n "$1" ]; then
+  DIR_NAME=$1
+else
+  echo "Error: No data file system mount path is provided."
+  echo "Usage: instance/storage/setup.sh <file_system_mount_path> <file_system_type [ xfs | ext4 ]> <target_volume_size_in_bytes> "
+  echo "Default file system type is ext4"
+  echo "If you skip <target_volume_size_in_bytes>, script will try to use the first unformatted volume ID."
+  echo "Usage example: instance/storage/setup.sh /data ext4 300000000000000"
+  exit 1
+fi
+
+# Create the directory if it doesn't exist
+if [ ! -d "$DIR_NAME" ]; then
+  echo "Creating directory $DIR_NAME"
+  mkdir -p "$DIR_NAME"
+fi
+
+# Case input for $2 between ext4 and xfs, use ext4 as default
+case $2 in
+  ext4)
+    echo "File system set to ext4"
+    FILE_SYSTEM="ext4"
+    FS_CONFIG="defaults"
+    ;;
+  xfs)
+    echo "File system set to xfs"
+    FILE_SYSTEM="xfs"
+    FS_CONFIG="noatime,nodiratime,nodiscard" # See more: https://cdrdv2-public.intel.com/686417/rocksdb-benchmark-tuning-guide-on-xeon.pdf
+    ;;
+  *)
+    echo "File system set to ext4"
+    FILE_SYSTEM="ext4"
+    FS_CONFIG="defaults"
+    ;;
+esac
+
+if [ -n "$3" ]; then
+  VOLUME_SIZE=$3
+else
+  echo "The size of volume for $DIR_NAME is not specified. Will try to guess volume ID."
+fi
+
+  echo "Checking if $DIR_NAME is mounted, and dont do anything if it is"
+  if [ $(df --output=target | grep -c "$DIR_NAME") -lt 1 ]; then
+
+  if [ -n "$VOLUME_SIZE" ]; then
+    VOLUME_ID=/dev/$(lsblk -lnb | awk -v VOLUME_SIZE_BYTES="$VOLUME_SIZE" '{if ($4== VOLUME_SIZE_BYTES) {print $1}}')
+    echo "Data volume size defined, use respective volume id: $VOLUME_ID"
+  else
+    VOLUME_ID=$(get_next_empty_nvme_disk)
+    echo "Data volume size undefined, trying volume id: $VOLUME_ID"
+  fi
+
+  attempts=0
+  until [ -n "$VOLUME_ID" ] || [ $attempts -ge 30 ]; do
+    echo "Waiting for data volume to appear..."
+    sleep 5
+    if [ -n "$VOLUME_SIZE" ]; then
+      VOLUME_ID=/dev/$(lsblk -lnb | awk -v VOLUME_SIZE_BYTES="$VOLUME_SIZE" '{if ($4== VOLUME_SIZE_BYTES) {print $1}}')
+    else
+      VOLUME_ID=$(get_next_empty_nvme_disk)
+    fi
+    attempts=$((attempts+1))
+  done
+
+  if [ -z "$VOLUME_ID" ]; then
+    echo "Error: Could not find a suitable volume to mount. Listing available disks:"
+    lsblk -lnb
+    exit 1
+  fi
+
+    echo "Formatting volume $VOLUME_ID with $FILE_SYSTEM"
+    make_fs $FILE_SYSTEM "$VOLUME_ID"
+    if [ $? -ne 0 ]; then
+      echo "Error: Failed to format the volume. Exiting."
+      exit 1
+    fi
+
+    sleep 10
+    VOLUME_UUID=$(lsblk -fn -o UUID  "$VOLUME_ID")
+    if [ -z "$VOLUME_UUID" ]; then
+      echo "Error: Could not get UUID for volume $VOLUME_ID. Exiting."
+      exit 1
+    fi
+    
+    VOLUME_FSTAB_CONF="UUID=$VOLUME_UUID $DIR_NAME $FILE_SYSTEM $FS_CONFIG 0 2"
+    echo "VOLUME_ID=$VOLUME_ID"
+    echo "VOLUME_UUID=$VOLUME_UUID"
+    echo "VOLUME_FSTAB_CONF=$VOLUME_FSTAB_CONF"
+
+    # Check if data disc is already in fstab and replace the line if it is with the new disc UUID
+    echo "Checking fstab for volume $DIR_NAME"
+    if [ $(grep -c "$DIR_NAME" /etc/fstab) -gt 0 ]; then
+      SED_REPLACEMENT_STRING="$(grep -n "$DIR_NAME" /etc/fstab | cut -d: -f1)s#.*#$VOLUME_FSTAB_CONF#"
+      # if file exists, delete it
+      if [ -f /etc/fstab.bak ]; then
+        rm /etc/fstab.bak
+      fi
+      cp /etc/fstab /etc/fstab.bak
+      sed -i "$SED_REPLACEMENT_STRING" /etc/fstab
+    else
+      echo "$VOLUME_FSTAB_CONF" | tee -a /etc/fstab
+    fi
+
+    echo "Mounting volume with mount -a"
+    mount -a
+    if [ $? -ne 0 ]; then
+      echo "Error: Failed to mount the volume. Exiting."
+      exit 1
+    fi
+    
+    echo "Volume mounted successfully at $DIR_NAME"
+  else
+    echo "$DIR_NAME volume is mounted, nothing changed"
+  fi