-
Notifications
You must be signed in to change notification settings - Fork 60
v2 -- Bitcoin Core Implementation #207
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
2484005
167f34c
071c1b5
a494a32
ef8f008
7739fa7
64104ee
7ad76fe
94e7d58
c78aab8
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
Original file line number | Diff line number | Diff line change | ||
---|---|---|---|---|
@@ -0,0 +1,31 @@ | ||||
#!/usr/bin/env node | ||||
import 'dotenv/config'; | ||||
import { App, Aspects } from 'aws-cdk-lib'; | ||||
import { AwsSolutionsChecks } from 'cdk-nag'; | ||||
import { BitcoinCommonStack } from './lib/common-infra'; | ||||
import { SingleNodeBitcoinCoreStack } from './lib/single-node-stack'; | ||||
import { HABitcoinCoreNodeStack } from './lib/ha-node-stack'; | ||||
import * as config from './lib/config/bitcoinConfig'; | ||||
|
||||
const app = new App(); | ||||
|
||||
Aspects.of(app).add(new AwsSolutionsChecks({ verbose: true })); | ||||
|
||||
const env = { | ||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please move processing of all environmental variables to |
||||
account: process.env.AWS_ACCOUNT_ID, | ||||
region: process.env.AWS_REGION, | ||||
}; | ||||
|
||||
const commonStack = new BitcoinCommonStack(app, 'BitcoinCommonStack', { env }); | ||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Pleas follow naming convention for stack names. Example:
|
||||
new SingleNodeBitcoinCoreStack(app, 'SingleNodeBitcoinCoreStack', { | ||||
env, | ||||
instanceRole: commonStack.instanceRole, | ||||
...config.baseNodeConfig, | ||||
}); | ||||
|
||||
new HABitcoinCoreNodeStack(app, 'HABitcoinCoreNodeStack', { | ||||
env, | ||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please explicitly set
|
||||
instanceRole: commonStack.instanceRole, | ||||
...config.baseNodeConfig, | ||||
...config.haNodeConfig, | ||||
}); |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,57 @@ | ||
{ | ||
"app": "npx ts-node --prefer-ts-exts app.ts", | ||
"watch": { | ||
"include": [ | ||
"**" | ||
], | ||
"exclude": [ | ||
"README.md", | ||
"cdk*.json", | ||
"**/*.d.ts", | ||
"**/*.js", | ||
"tsconfig.json", | ||
"package*.json", | ||
"yarn.lock", | ||
"node_modules", | ||
"test" | ||
] | ||
}, | ||
"context": { | ||
"@aws-cdk/aws-lambda:recognizeLayerVersion": true, | ||
"@aws-cdk/core:checkSecretUsage": true, | ||
"@aws-cdk/core:target-partitions": [ | ||
"aws", | ||
"aws-cn" | ||
], | ||
"@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, | ||
"@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, | ||
"@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, | ||
"@aws-cdk/aws-iam:minimizePolicies": true, | ||
"@aws-cdk/core:validateSnapshotRemovalPolicy": true, | ||
"@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, | ||
"@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, | ||
"@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, | ||
"@aws-cdk/aws-apigateway:disableCloudWatchRole": true, | ||
"@aws-cdk/core:enablePartitionLiterals": true, | ||
"@aws-cdk/aws-events:eventsTargetQueueSameAccount": true, | ||
"@aws-cdk/aws-iam:standardizedServicePrincipals": true, | ||
"@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true, | ||
"@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true, | ||
"@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true, | ||
"@aws-cdk/aws-route53-patters:useCertificate": true, | ||
"@aws-cdk/customresources:installLatestAwsSdkDefault": false, | ||
"@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true, | ||
"@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true, | ||
"@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true, | ||
"@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true, | ||
"@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true, | ||
"@aws-cdk/aws-redshift:columnId": true, | ||
"@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true, | ||
"@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true, | ||
"@aws-cdk/aws-apigateway:requestValidatorUniqueId": true, | ||
"@aws-cdk/aws-kms:aliasNameRef": true, | ||
"@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true, | ||
"@aws-cdk/core:includePrefixInUniqueNameGeneration": true, | ||
"@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true | ||
} | ||
} |
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is a command line script and uses generic "crypto" module. Choosing to use JavaScript for this script introduces extra dependencies to the global OR local There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I understand that "introducing extra dependencies to the global or local However, it is absolutely necessary to incorporate the Secrets Manager dependency, as the generateRPCAuth script requires it to securely store the credentials which are needed to access the node. (To my understanding, this is the only Blueprint that requires credentials for performing external RPC Requests ) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. FYI, the The above links to this script , which I expanded upon. Though it can be rewritten in bash, I do not believe it is worth my time or effort to do so. I am closing this. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We have successfully maintained our protocol of avoiding additional dependencies in both global and local package.json files across all blueprints. Please continue adhering to this standard by:
Please note that this requirement serves as a critical blocking issue for this blueprint. The inclusion of new dependencies will only be considered if there is substantial evidence demonstrating that:
Before proceeding, please ensure these criteria are thoroughly evaluated and documented. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please note that There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Noted with thanks. The script includes "@aws-sdk/client-secrets-manager" and "base64url" that are also added as dependencies to the root package.json file, and that is a problem. See our conversation here: https://github.com/aws-samples/aws-blockchain-node-runners/pull/207/files/c78aab85e13997f02078bfedd442835b22ec098c#r2244133508 The reason I brought up "crypto" in the original message is to highlight that there is no Bitcoin-specific cryptography involved in this logic. Therefore, there is a potential to use OpenSSL to do the same in Bash. This needs to be researched. |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,100 @@ | ||
const crypto = require('crypto'); | ||
const base64url = require('base64url'); | ||
const fs = require('fs'); | ||
const { SecretsManagerClient, CreateSecretCommand, PutSecretValueCommand } = require('@aws-sdk/client-secrets-manager'); | ||
|
||
// Set up AWS SDK client | ||
const client = new SecretsManagerClient({ region: 'us-east-1' }); // Change region if needed | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. AWS region is hard coded. Please make this configurable as the script can be used in different regions. |
||
|
||
// Create size byte hex salt | ||
function genSalt(size = 16) { | ||
const buffer = crypto.randomBytes(size); | ||
return buffer.toString('hex'); | ||
} | ||
|
||
// Create 32 byte b64 password | ||
function genPass(size = 32) { | ||
const buffer = crypto.randomBytes(size); | ||
return base64url.fromBase64(buffer.toString('base64')); | ||
} | ||
|
||
function genUser() { | ||
return 'user_' + Math.round(Math.random() * 1000); | ||
} | ||
|
||
function genHash(password, salt) { | ||
const hash = crypto | ||
.createHmac('sha256', salt) | ||
.update(password) | ||
.digest('hex'); | ||
return hash; | ||
} | ||
|
||
function genRpcAuth(username = genUser(), password = genPass(), salt = genSalt()) { | ||
const hash = genHash(password, salt); | ||
return { username, password, salt, hash }; | ||
} | ||
|
||
function writeRpcAuthToConf(rpcauthStr) { | ||
const confPath = 'lib/bitcoin.conf'; | ||
try { | ||
fs.writeFileSync(confPath, rpcauthStr + '\n', { flag: 'a' }); | ||
console.log(`Successfully wrote to ${confPath}`); | ||
} catch (error) { | ||
console.error(`Error writing to ${confPath}:`, error); | ||
} | ||
} | ||
|
||
async function storeCredentialsInAWS(username, password) { | ||
const secretName = 'bitcoin_rpc_credentials'; | ||
const secretValue = `${username}:${password}`; | ||
|
||
try { | ||
const createCommand = new CreateSecretCommand({ | ||
Name: secretName, | ||
SecretString: secretValue, | ||
}); | ||
await client.send(createCommand); | ||
console.log(`Successfully stored credentials in AWS Secrets Manager: ${secretName}`); | ||
} catch (error) { | ||
if (error.name === 'ResourceExistsException') { | ||
const updateCommand = new PutSecretValueCommand({ | ||
SecretId: secretName, | ||
SecretString: secretValue, | ||
}); | ||
await client.send(updateCommand); | ||
console.log(`Successfully updated existing secret in AWS Secrets Manager: ${secretName}`); | ||
} else { | ||
console.error(`Error storing credentials in AWS Secrets Manager:`, error); | ||
} | ||
} | ||
} | ||
|
||
async function genRpcAuthStr(username, password, salt) { | ||
const rpcauth = genRpcAuth(username, password, salt); | ||
const str = `rpcauth=${rpcauth.username}:${rpcauth.salt}$${rpcauth.hash}`; | ||
const strEscapeCharacter = `${rpcauth.username}:${rpcauth.salt}\\$${rpcauth.hash}`; | ||
console.log(`Username: ${rpcauth.username}`); | ||
console.log("Password generated securely and stored in Secrets Manager"); | ||
console.log(`rpcauth string with escape character: ${strEscapeCharacter}`); // Print the rpcauth string | ||
|
||
// Write to bitcoin.conf | ||
writeRpcAuthToConf(str); | ||
|
||
// Store in AWS Secrets Manager | ||
await storeCredentialsInAWS(rpcauth.username, rpcauth.password); | ||
|
||
return str; | ||
} | ||
|
||
// Example usage | ||
genRpcAuthStr(); | ||
|
||
module.exports = { | ||
genSalt, | ||
genPass, | ||
genUser, | ||
genHash, | ||
genRpcAuth, | ||
genRpcAuthStr, | ||
}; |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
module.exports = { | ||
testEnvironment: 'node', | ||
roots: ['<rootDir>/test'], | ||
testMatch: ['**/*.test.ts'], | ||
transform: { | ||
'^.+\\.tsx?$': 'ts-jest' | ||
} | ||
}; |
Original file line number | Diff line number | Diff line change | ||
---|---|---|---|---|
@@ -0,0 +1,62 @@ | ||||
#!/bin/bash | ||||
# This script is used to set up a mainnet Bitcoin Core node on an Amazon Linux 2 instance. | ||||
set -euo pipefail | ||||
|
||||
# The stack passes lifecycle hook information when used in an Auto | ||||
# Scaling Group. If no lifecycle hook name is supplied (single node | ||||
# deployment), default to "none" and use CloudFormation signaling. | ||||
LIFECYCLE_HOOK_NAME=${LIFECYCLE_HOOK_NAME:-none} | ||||
AUTOSCALING_GROUP_NAME=${AUTOSCALING_GROUP_NAME:-none} | ||||
|
||||
# Ensure the data volume is mounted before proceeding | ||||
until mountpoint -q /home/bitcoin; do | ||||
echo "Waiting for /home/bitcoin to be mounted..." | ||||
sleep 2 | ||||
done | ||||
|
||||
yum update -y | ||||
amazon-linux-extras install docker -y | ||||
service docker start | ||||
systemctl enable docker | ||||
|
||||
# Create bitcoin user with specific UID:GID to match the container's expected values | ||||
if id -u bitcoin > /dev/null 2>&1; then | ||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. In all bluepirints user and group names are
|
||||
# User exists, update to correct UID:GID | ||||
usermod -u 101 bitcoin | ||||
groupmod -g 101 bitcoin | ||||
else | ||||
# Create user with specific UID:GID | ||||
groupadd -g 101 bitcoin | ||||
useradd -u 101 -g 101 -m -s /bin/bash bitcoin | ||||
fi | ||||
|
||||
# Create the bitcoin data directory structure on the mounted EBS volume | ||||
mkdir -p /home/bitcoin/.bitcoin | ||||
echo "${BITCOIN_CONF}" > /home/bitcoin/.bitcoin/bitcoin.conf | ||||
|
||||
# Set proper permissions for the Bitcoin configuration | ||||
chown -R bitcoin:bitcoin /home/bitcoin | ||||
chmod -R 755 /home/bitcoin | ||||
|
||||
# Run Bitcoin Core in Docker with proper volume mapping | ||||
# Modified to ensure data is stored on the EBS volume | ||||
docker run -d --name bitcoind \ | ||||
-v /home/bitcoin/.bitcoin:/home/bitcoin/.bitcoin \ | ||||
-p 8333:8333 \ | ||||
-p 8332:8332 \ | ||||
--restart unless-stopped \ | ||||
bitcoin/bitcoin:latest \ | ||||
bitcoind -datadir=/home/bitcoin/.bitcoin | ||||
|
||||
# Signal completion depending on deployment type | ||||
if [[ "$LIFECYCLE_HOOK_NAME" != "none" ]]; then | ||||
echo "Signaling ASG lifecycle hook to complete" | ||||
TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") | ||||
INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/instance-id) | ||||
aws autoscaling complete-lifecycle-action --lifecycle-action-result CONTINUE --instance-id "$INSTANCE_ID" --lifecycle-hook-name "$LIFECYCLE_HOOK_NAME" --auto-scaling-group-name "$AUTOSCALING_GROUP_NAME" --region "$AWS_REGION" | ||||
else | ||||
if ! command -v cfn-signal &> /dev/null; then | ||||
yum install -y aws-cfn-bootstrap | ||||
fi | ||||
cfn-signal --stack "$STACK_NAME" --resource "$RESOURCE_ID" --region "$AWS_REGION" | ||||
fi |
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please follow the There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please make sure the script reports at least two metrics: |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
#!/bin/bash | ||
# This script is used to set up a cron job to send the Bitcoin block height to Amazon CloudWatch every 5 minutes. | ||
REGION=${AWS_REGION} | ||
(crontab -l 2>/dev/null; echo "*/5 * * * * sudo /usr/bin/docker exec bitcoind bitcoin-cli getblockcount | xargs -I {} sudo /usr/bin/aws cloudwatch put-metric-data --metric-name BlockHeight --namespace Bitcoin --unit Count --value {} --region $REGION") | crontab - |
Original file line number | Diff line number | Diff line change | ||
---|---|---|---|---|
@@ -0,0 +1,30 @@ | ||||
#!/bin/bash | ||||
# This script is used to set up the Amazon CloudWatch agent on an Amazon Linux 2 instance. | ||||
|
||||
yum install -y amazon-cloudwatch-agent | ||||
mkdir -p /opt/aws/amazon-cloudwatch-agent/etc | ||||
cat <<EOF > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json | ||||
{ | ||||
"metrics": { | ||||
"metrics_collected": { | ||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please add reporting for Disc IO. Example:
|
||||
"disk": { | ||||
"measurement": ["used_percent", "inodes_free"], | ||||
"resources": ["*"], | ||||
"ignore_file_system_types": ["sysfs", "devtmpfs"] | ||||
}, | ||||
"mem": { | ||||
"measurement": ["mem_used_percent"] | ||||
}, | ||||
"cpu": { | ||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please add "cpu_usage_iowait" |
||||
"measurement": ["cpu_usage_idle", "cpu_usage_user", "cpu_usage_system"] | ||||
}, | ||||
"net": { | ||||
"measurement": ["net_bytes_sent", "net_bytes_recv"], | ||||
"resources": ["eth0"] | ||||
} | ||||
} | ||||
} | ||||
} | ||||
EOF | ||||
|
||||
/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Please add the following two parameters to security check:
reports: true,
logIgnores: false,
Example:
aws-blockchain-node-runners/lib/base/app.ts
Line 56 in 5c8636c