Skip to content

Commit 945df88

Browse files
AhmetFurkanDEMIRAhmetFurkanDEMIR
committed
kafka connect nodes
1 parent 64a60ed commit 945df88

File tree

8 files changed

+33
-21
lines changed

8 files changed

+33
-21
lines changed

‎kafka_connect/generate-inventory.sh‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,13 @@ if [ ${#CONNECT_DNS_ARRAY[@]} -ne 2 ]; then
4444
echo "⚠️ Warning: Expected 2 Kafka Connect nodes, found ${#CONNECT_DNS_ARRAY[@]}"
4545
fi
4646

47+
# Calculate CONNECT_PUBLIC_IP from DNS (convert ec2-54-217-66-144 to 54.217.66.144)
48+
CONNECT_NODE1_DNS="${CONNECT_DNS_ARRAY[0]}"
49+
CONNECT_NODE1_IP=$(echo "${CONNECT_NODE1_DNS}" | sed 's/ec2-//' | cut -d'.' -f1 | tr '-' '.')
50+
51+
CONNECT_NODE2_DNS="${CONNECT_DNS_ARRAY[1]}"
52+
CONNECT_NODE2_IP=$(echo "${CONNECT_NODE2_DNS}" | sed 's/ec2-//' | cut -d'.' -f1 | tr '-' '.')
53+
4754
# Extract Kafka Broker Public DNS Names
4855
BROKER_PUBLIC_DNS=$(echo "$STATE_JSON" | jq -r '.outputs.kafka_broker_public_dns.value[]' 2>/dev/null || echo "")
4956
if [ -z "$BROKER_PUBLIC_DNS" ] || [ "$BROKER_PUBLIC_DNS" = "null" ]; then
@@ -110,12 +117,12 @@ SSL_KEYSTORE_PASSWORD=${SSL_KEYSTORE_PASSWORD:-confluentkeystorestorepass}
110117
SSL_KEY_PASSWORD=${SSL_KEY_PASSWORD:-confluentkeystorestorepass}
111118
112119
# Kafka Connect Configuration (Node 1 as primary for connector config)
113-
CONNECT_PUBLIC_DNS=${CONNECT_DNS_ARRAY[0]}
114-
CONNECT_PUBLIC_IP=\$(echo "${CONNECT_DNS_ARRAY[0]}" | cut -d'-' -f2- | tr '-' '.')
120+
CONNECT_PUBLIC_DNS=${CONNECT_NODE1_DNS}
121+
CONNECT_PUBLIC_IP=${CONNECT_NODE1_IP}
115122
116123
# Kafka Connect Node 2
117-
CONNECT_PUBLIC_DNS_NODE2=${CONNECT_DNS_ARRAY[1]}
118-
CONNECT_PUBLIC_IP_NODE2=\$(echo "${CONNECT_DNS_ARRAY[1]}" | cut -d'-' -f2- | tr '-' '.')
124+
CONNECT_PUBLIC_DNS_NODE2=${CONNECT_NODE2_DNS}
125+
CONNECT_PUBLIC_IP_NODE2=${CONNECT_NODE2_IP}
119126
120127
# JWT Token for FastAPI Authentication
121128
JWT_TOKEN=${JWT_TOKEN:-token}
@@ -133,12 +140,12 @@ if [ -f "${CONNECTOR_TEMPLATE}" ]; then
133140
JWT_TOKEN_VALUE=${JWT_TOKEN:-token}
134141

135142
# Replace placeholders in template
136-
sed -e "s|{{CONNECT_PUBLIC_IP}}|${CONNECT_PUBLIC_DNS}|g" \
143+
sed -e "s|{{CONNECT_PUBLIC_IP}}|${CONNECT_NODE1_DNS}|g" \
137144
-e "s|{{JWT_TOKEN}}|${JWT_TOKEN_VALUE}|g" \
138145
"${CONNECTOR_TEMPLATE}" > "${CONNECTOR_JSON}"
139146

140147
echo "✅ Connector JSON generated from template"
141-
echo " URL: http://${CONNECT_PUBLIC_DNS}:2020/topics"
148+
echo " URL: http://${CONNECT_NODE1_DNS}:2020/topics"
142149
echo ""
143150
fi
144151

@@ -155,7 +162,7 @@ if [ -d "${SCRIPT_DIR}/scripts" ]; then
155162

156163
# Update KAFKA_BOOTSTRAP_SERVERS and CONNECT_HOST
157164
sed -i "s|KAFKA_BOOTSTRAP_SERVERS=.*|KAFKA_BOOTSTRAP_SERVERS=\"${KAFKA_BOOTSTRAP_SERVERS}\"|g" "$script"
158-
sed -i "s|CONNECT_HOST=.*|CONNECT_HOST=\"${CONNECT_PUBLIC_DNS}\"|g" "$script"
165+
sed -i "s|CONNECT_HOST=.*|CONNECT_HOST=\"${CONNECT_NODE1_DNS}\"|g" "$script"
159166

160167
echo " ✅ Updated $(basename $script)"
161168
fi

‎kafka_connect/inventory.yml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,13 @@ all:
33
children:
44
kafka_connect:
55
hosts:
6-
kafka-connect-node:
7-
ansible_host: ec2-34-240-197-144.eu-west-1.compute.amazonaws.com
6+
kafka-connect-node-1:
7+
ansible_host: ec2-54-217-66-144.eu-west-1.compute.amazonaws.com
8+
ansible_user: ubuntu
9+
ansible_ssh_private_key_file: ~/.ssh/trendyol-kafka-key.pem
10+
ansible_ssh_common_args: '-o StrictHostKeyChecking=no'
11+
kafka-connect-node-2:
12+
ansible_host: ec2-34-243-2-32.eu-west-1.compute.amazonaws.com
813
ansible_user: ubuntu
914
ansible_ssh_private_key_file: ~/.ssh/trendyol-kafka-key.pem
1015
ansible_ssh_common_args: '-o StrictHostKeyChecking=no'

‎kafka_connect/scripts/create-connector.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,10 @@ ENV_FILE="${SCRIPT_DIR}/../.env"
1010

1111
if [ -f "$ENV_FILE" ]; then
1212
source "$ENV_FILE"
13-
CONNECT_HOST="ec2-34-240-197-144.eu-west-1.compute.amazonaws.com"
13+
CONNECT_HOST="ec2-54-217-66-144.eu-west-1.compute.amazonaws.com"
1414
else
1515
echo "⚠️ Warning: .env file not found, using default value"
16-
CONNECT_HOST="ec2-34-240-197-144.eu-west-1.compute.amazonaws.com"
16+
CONNECT_HOST="ec2-54-217-66-144.eu-west-1.compute.amazonaws.com"
1717
fi
1818

1919
CONNECTOR_CONFIG="${CONNECTOR_CONFIG:-${SCRIPT_DIR}/../connectors/http-source-connector.json}"

‎kafka_connect/scripts/delete-connector.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,10 @@ ENV_FILE="${SCRIPT_DIR}/../.env"
1010

1111
if [ -f "$ENV_FILE" ]; then
1212
source "$ENV_FILE"
13-
CONNECT_HOST="ec2-34-240-197-144.eu-west-1.compute.amazonaws.com"
13+
CONNECT_HOST="ec2-54-217-66-144.eu-west-1.compute.amazonaws.com"
1414
else
1515
echo "⚠️ Warning: .env file not found, using default value"
16-
CONNECT_HOST="ec2-34-240-197-144.eu-west-1.compute.amazonaws.com"
16+
CONNECT_HOST="ec2-54-217-66-144.eu-west-1.compute.amazonaws.com"
1717
fi
1818

1919
CONNECTOR_NAME="${1:-http-source-topics-connector}"

‎kafka_connect/scripts/restart-task.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,10 @@ ENV_FILE="${SCRIPT_DIR}/../.env"
1010

1111
if [ -f "$ENV_FILE" ]; then
1212
source "$ENV_FILE"
13-
CONNECT_HOST="ec2-34-240-197-144.eu-west-1.compute.amazonaws.com"
13+
CONNECT_HOST="ec2-54-217-66-144.eu-west-1.compute.amazonaws.com"
1414
else
1515
echo "⚠️ Warning: .env file not found, using default value"
16-
CONNECT_HOST="ec2-34-240-197-144.eu-west-1.compute.amazonaws.com"
16+
CONNECT_HOST="ec2-54-217-66-144.eu-west-1.compute.amazonaws.com"
1717
fi
1818

1919
CONNECTOR_NAME="${1:-http-source-topics-connector}"

‎kafka_connect/scripts/setup-kafka-connect.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,12 @@ ENV_FILE="${KAFKA_CONNECT_DIR}/.env"
1414

1515
if [ -f "$ENV_FILE" ]; then
1616
source "$ENV_FILE"
17-
CONNECT_HOST="${CONNECT_PUBLIC_DNS:-localhost}"
17+
CONNECT_HOST="ec2-54-217-66-144.eu-west-1.compute.amazonaws.com"
1818
echo "📋 Loaded configuration from .env"
1919
echo " Connect Host: ${CONNECT_HOST}"
2020
else
2121
echo "⚠️ Warning: .env file not found at ${ENV_FILE}"
22-
CONNECT_HOST="localhost"
22+
CONNECT_HOST="ec2-54-217-66-144.eu-west-1.compute.amazonaws.com"
2323
fi
2424
echo ""
2525

‎kafka_connect/scripts/test-rest-api.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,10 @@ ENV_FILE="${SCRIPT_DIR}/../.env"
1111

1212
if [ -f "$ENV_FILE" ]; then
1313
source "$ENV_FILE"
14-
CONNECT_HOST="ec2-34-240-197-144.eu-west-1.compute.amazonaws.com"
14+
CONNECT_HOST="ec2-54-217-66-144.eu-west-1.compute.amazonaws.com"
1515
else
1616
echo "⚠️ Warning: .env file not found, using default value"
17-
CONNECT_HOST="ec2-34-240-197-144.eu-west-1.compute.amazonaws.com"
17+
CONNECT_HOST="ec2-54-217-66-144.eu-west-1.compute.amazonaws.com"
1818
fi
1919

2020
CONNECTOR_NAME="${CONNECTOR_NAME:-http-source-topics-connector}"

‎kafka_connect/scripts/update-connector.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,10 @@ ENV_FILE="${SCRIPT_DIR}/../.env"
1010

1111
if [ -f "$ENV_FILE" ]; then
1212
source "$ENV_FILE"
13-
CONNECT_HOST="ec2-34-240-197-144.eu-west-1.compute.amazonaws.com"
13+
CONNECT_HOST="ec2-54-217-66-144.eu-west-1.compute.amazonaws.com"
1414
else
1515
echo "⚠️ Warning: .env file not found, using default value"
16-
CONNECT_HOST="ec2-34-240-197-144.eu-west-1.compute.amazonaws.com"
16+
CONNECT_HOST="ec2-54-217-66-144.eu-west-1.compute.amazonaws.com"
1717
fi
1818

1919
CONNECTOR_NAME="${1:-http-source-topics-connector}"

0 commit comments

Comments
 (0)