Skip to content

Commit 57be4ca

Browse files
committed
Solana. Outbound network throlling test
1 parent d0fab74 commit 57be4ca

12 files changed

+154
-1
lines changed
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/bin/bash
2+
3+
# Add input as command line parameters for name of the directory to mount
4+
if [ -n "$1" ]; then
5+
LIMIT_OUT_TRAFFIC_MBPS=$1
6+
else
7+
echo "Warning: Specify max value for outbound data traffic in Mbps."
8+
echo "Usage: net-rules.sh <max_bandwidth_mbps>"
9+
echo "Default is 26"
10+
LIMIT_OUT_TRAFFIC_MBPS=26
11+
fi
12+
13+
# Step 1: Create an iptables rule to mark packets going to public IPs
14+
# Create a new chain for our marking rules
15+
iptables -t mangle -N MARKING
16+
17+
# Add rules to return (skip marking) for private IP ranges
18+
iptables -t mangle -A MARKING -d 10.0.0.0/8 -j RETURN
19+
iptables -t mangle -A MARKING -d 172.16.0.0/12 -j RETURN
20+
iptables -t mangle -A MARKING -d 192.168.0.0/16 -j RETURN
21+
iptables -t mangle -A MARKING -d 169.254.0.0/16 -j RETURN
22+
23+
# Mark remaining traffic (public IPs)
24+
iptables -t mangle -A MARKING -j MARK --set-mark 1
25+
26+
# Jump to our MARKING chain from OUTPUT
27+
iptables -t mangle -A OUTPUT -j MARKING
28+
29+
# Step 2: Set up tc with filter for marked packets
30+
INTERFACE=$(ip -br addr show | grep -v '^lo' | awk '{print $1}' | head -n1)
31+
32+
tc qdisc add dev $INTERFACE root handle 1: prio
33+
34+
# Step 3: Add the tbf filter for marked packets
35+
tc filter add dev $INTERFACE parent 1: protocol ip handle 1 fw flowid 1:1
36+
tc qdisc add dev $INTERFACE parent 1:1 tbf rate "${LIMIT_OUT_TRAFFIC_MBPS}mbit" burst 20kb latency 50ms
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
3+
# Remove tc rules
4+
/usr/sbin/tc qdisc del dev eth0 root
5+
6+
# Remove iptables rules
7+
/usr/sbin/iptables -t mangle -D OUTPUT -j MARKING
8+
/usr/sbin/iptables -t mangle -F MARKING
9+
/usr/sbin/iptables -t mangle -X MARKING
10+
11+
exit 0;
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
[Unit]
2+
Description=ipables and Traffic Control Rules
3+
After=network.target
4+
5+
[Service]
6+
Type=oneshot
7+
RemainAfterExit=yes
8+
ExecStart=/opt/instance/network/net-rules-start.sh _LIMIT_OUT_TRAFFIC_MBPS_
9+
ExecStop=/opt/instance/network/net-rules-stop.sh
10+
11+
[Install]
12+
WantedBy=multi-user.target
13+
EOF
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
[Unit]
2+
Description="Net sync checker for blockchain node"
3+
4+
[Service]
5+
ExecStart=/opt/instance/network/net-syncchecker.sh
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
[Unit]
2+
Description="Run Network Sync checker service every 5 min"
3+
4+
[Timer]
5+
OnCalendar=*:*:0/5
6+
Unit=net-sync-checker.service
7+
8+
[Install]
9+
WantedBy=multi-user.target
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
3+
INIT_COMPLETED_FILE=/data/data/init-completed
4+
5+
TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
6+
EC2_INTERNAL_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/local-ipv4)
7+
8+
if [ -f "$INIT_COMPLETED_FILE" ]; then
9+
SOLANA_SLOTS_BEHIND_DATA=$(curl -s -X POST -H "Content-Type: application/json" -d ' {"jsonrpc":"2.0","id":1, "method":"getHealth"}' http://$EC2_INTERNAL_IP:8899 | jq .error.data)
10+
SOLANA_SLOTS_BEHIND=$(echo $SOLANA_SLOTS_BEHIND_DATA | jq .numSlotsBehind -r)
11+
12+
if [ "$SOLANA_SLOTS_BEHIND" == "null" ] || [ -z "$SOLANA_SLOTS_BEHIND" ]
13+
then
14+
SOLANA_SLOTS_BEHIND=0
15+
fi
16+
17+
if [ $SOLANA_SLOTS_BEHIND -gt 100 ]
18+
then
19+
if systemctl is-active --quiet net-rules; then
20+
systemctl stop net-rules
21+
fi
22+
fi
23+
24+
if [ $SOLANA_SLOTS_BEHIND -eq 0 ]
25+
then
26+
if ! systemctl is-active --quiet net-rules; then
27+
systemctl start net-rules
28+
fi
29+
fi
30+
fi
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/bin/bash
2+
3+
# Add input as command line parameters for name of the directory to mount
4+
if [ -n "$1" ]; then
5+
LIMIT_OUT_TRAFFIC_MBPS=$1
6+
else
7+
echo "Warning: Specify max value for outbound data traffic in Mbps."
8+
echo "Usage: instance/network/setup.sh <max_bandwidth_mbps>"
9+
LIMIT_OUT_TRAFFIC_MBPS=26
10+
fi
11+
12+
INTERFACE=$(ip -br addr show | grep -v '^lo' | awk '{print $1}' | head -n1)
13+
NET_SCRIPTS_PATH="/opt/instance/network"
14+
15+
# Replace _LIMIT_OUT_TRAFFIC_MBPS_ with the value of LIMIT_OUT_TRAFFIC_MBPS in file /opt/network/net-rules.service.template
16+
sed -i "s/_LIMIT_OUT_TRAFFIC_MBPS_/${LIMIT_OUT_TRAFFIC_MBPS}/g" $NET_SCRIPTS_PATH/net-rules.service
17+
sed -i "s/_INTERFACE_/${INTERFACE}/g" $NET_SCRIPTS_PATH/net-rules.service
18+
19+
# Copy the file $NET_SCRIPTS_PATH/net-rules.service to /etc/systemd/system/net-rules.service
20+
cp $NET_SCRIPTS_PATH/net-rules.service /etc/systemd/system/net-rules.service
21+
22+
echo "Enabling net rules service"
23+
systemctl enable net-rules.service
24+
25+
echo "Setting up sync-checker service"
26+
mv $NET_SCRIPTS_PATH/net-sync-checker.service /etc/systemd/system/net-sync-checker.service
27+
28+
# Run every 5 minutes
29+
echo "Setting up sync-checker timer"
30+
mv $NET_SCRIPTS_PATH/net-sync-checker.timer /etc/systemd/system/net-sync-checker.timer
31+
32+
echo "Starting net sync checker timer"
33+
systemctl start net-sync-checker.timer
34+
systemctl enable net-sync-checker.timer

lib/solana/lib/assets/user-data-ubuntu.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ chmod 600 /etc/cdk_environment
2222
echo "SOLANA_CLUSTER=${_SOLANA_CLUSTER_}"
2323
echo "LIFECYCLE_HOOK_NAME=${_LIFECYCLE_HOOK_NAME_}"
2424
echo "ASG_NAME=${_ASG_NAME_}"
25+
echo "LIMIT_OUT_TRAFFIC_MBPS=${_LIMIT_OUT_TRAFFIC_MBPS_}"
2526
} >> /etc/cdk_environment
2627
source /etc/cdk_environment
2728

@@ -127,6 +128,11 @@ systemctl restart amazon-cloudwatch-agent
127128

128129
systemctl daemon-reload
129130

131+
if [[ "$LIMIT_OUT_TRAFFIC_MBPS" == "true" ]]; then
132+
echo "Limiting out traffic"
133+
/opt/instance/network/setup.sh
134+
fi
135+
130136
echo "Starting up the node service"
131137
systemctl enable --now node
132138

lib/solana/lib/config/node-config.interface.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ export interface SolanaBaseNodeConfig extends configTypes.BaseNodeConfig {
2222
voteAccountSecretARN: string;
2323
authorizedWithdrawerAccountSecretARN: string;
2424
registrationTransactionFundingAccountSecretARN: string;
25+
limitOutTrafficMbps: number;
2526
}
2627

2728
export interface SolanaHAConfig {

lib/solana/lib/config/node-config.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ export const baseNodeConfig: configTypes.SolanaBaseNodeConfig = {
3434
instanceType: new ec2.InstanceType(process.env.SOLANA_INSTANCE_TYPE ? process.env.SOLANA_INSTANCE_TYPE : "r6a.8xlarge"),
3535
instanceCpuType: process.env.SOLANA_CPU_TYPE?.toLowerCase() == "x86_64" ? ec2.AmazonLinuxCpuType.X86_64 : ec2.AmazonLinuxCpuType.ARM_64,
3636
solanaCluster: <configTypes.SolanaCluster> process.env.SOLANA_CLUSTER || "mainnet-beta",
37-
solanaVersion: validateVersion(process.env.SOLANA_VERSION) || "2.0.18",
37+
solanaVersion: validateVersion(process.env.SOLANA_VERSION) || "2.1.16",
3838
nodeConfiguration: <configTypes.SolanaNodeConfiguration> process.env.SOLANA_NODE_CONFIGURATION || "baserpc",
3939
dataVolume: {
4040
sizeGiB: process.env.SOLANA_DATA_VOL_SIZE ? parseInt(process.env.SOLANA_DATA_VOL_SIZE): 2000,
@@ -52,6 +52,7 @@ export const baseNodeConfig: configTypes.SolanaBaseNodeConfig = {
5252
voteAccountSecretARN: process.env.SOLANA_VOTE_ACCOUNT_SECRET_ARN || "none",
5353
authorizedWithdrawerAccountSecretARN: process.env.SOLANA_AUTHORIZED_WITHDRAWER_ACCOUNT_SECRET_ARN || "none",
5454
registrationTransactionFundingAccountSecretARN: process.env.SOLANA_REGISTRATION_TRANSACTION_FUNDING_ACCOUNT_SECRET_ARN || "none",
55+
limitOutTrafficMbps: process.env.SOLANA_LIMIT_OUT_TRAFFIC_MBPS ? parseInt(process.env.SOLANA_LIMIT_OUT_TRAFFIC_MBPS) : 25,
5556
};
5657

5758
export const haNodeConfig: configTypes.SolanaHAConfig = {

0 commit comments

Comments
 (0)