[improve] use pulsar-client-go-test docker to run blue-green cluster test (#1287)

Author: geniusjoe

Makefile

@@ -53,27 +53,25 @@ container:
 test: container test_standalone test_clustered test_extensible_load_manager
 
 test_standalone: container
-	docker run -v /var/run/docker.sock:/var/run/docker.sock -i ${IMAGE_NAME} bash -c "cd /pulsar/pulsar-client-go && ./scripts/run-ci.sh"
+	docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -i ${IMAGE_NAME} bash -c "cd /pulsar/pulsar-client-go && ./scripts/run-ci.sh"
 
 test_clustered: container
 	PULSAR_VERSION=${PULSAR_VERSION} docker compose -f integration-tests/clustered/docker-compose.yml up -d
 	until curl http://localhost:8080/metrics > /dev/null 2>&1; do sleep 1; done
-	docker run --network "clustered_pulsar" -i ${IMAGE_NAME} bash -c "cd /pulsar/pulsar-client-go && ./scripts/run-ci-clustered.sh"
+	docker run --rm --network "clustered_pulsar" -i ${IMAGE_NAME} bash -c "cd /pulsar/pulsar-client-go && ./scripts/run-ci-clustered.sh"
 	PULSAR_VERSION=${PULSAR_VERSION} docker compose -f integration-tests/clustered/docker-compose.yml down
 
 test_extensible_load_manager: container
 	PULSAR_VERSION=${PULSAR_VERSION} docker compose -f integration-tests/extensible-load-manager/docker-compose.yml up -d
 	until curl http://localhost:8080/metrics > /dev/null 2>&1; do sleep 1; done
-	docker run --network "extensible-load-manager_pulsar" -i ${IMAGE_NAME} bash -c "cd /pulsar/pulsar-client-go && ./scripts/run-ci-extensible-load-manager.sh"
+	docker run --rm --network "extensible-load-manager_pulsar" -i ${IMAGE_NAME} bash -c "cd /pulsar/pulsar-client-go && ./scripts/run-ci-extensible-load-manager.sh"
 
 	PULSAR_VERSION=${PULSAR_VERSION} docker compose -f integration-tests/blue-green/docker-compose.yml up -d
 	until curl http://localhost:8081/metrics > /dev/null 2>&1 ; do sleep 1; done
 
-	# run blue-green migration test (run this test from this env to access both clusters)
-	go test -race -coverprofile=/tmp/coverage-blue_green_topic_migration -timeout=5m -tags extensible_load_manager -v -run TestBlueGreenMigrationTestSuite ./pulsar
-	go tool cover -html=/tmp/coverage-blue_green_topic_migration -o coverage-blue_green_topic_migration.html
-	PULSAR_VERSION=${PULSAR_VERSION} docker compose -f integration-tests/extensible-load-manager/docker-compose.yml down
+	docker run --rm --network="extensible-load-manager_pulsar" -i ${IMAGE_NAME} bash -c "cd /pulsar/pulsar-client-go && ./scripts/run-ci-blue-green-cluster.sh"
 	PULSAR_VERSION=${PULSAR_VERSION} docker compose -f integration-tests/blue-green/docker-compose.yml down
+	PULSAR_VERSION=${PULSAR_VERSION} docker compose -f integration-tests/extensible-load-manager/docker-compose.yml down
 
 clean:
 	docker rmi --force $(IMAGE_NAME) || true

integration-tests/blue-green/docker-compose.yml

@@ -17,18 +17,20 @@
 
 version: '3'
 networks:
-  green-pulsar:
-    driver: bridge
+  # We use external extensible-load-manager_pulsar docker network
+  # to make sure blue cluster and green cluster are in the same network
+  extensible-load-manager_pulsar:
+    external: true
 services:
   # Start ZooKeeper
-  zookeeper:
+  green-zookeeper:
     image: apachepulsar/pulsar:${PULSAR_VERSION}
     container_name: green-zookeeper
     restart: on-failure
     networks:
-      - green-pulsar
+      - extensible-load-manager_pulsar
     environment:
-      - metadataStoreUrl=zk:zookeeper:2181
+      - metadataStoreUrl=zk:green-zookeeper:2181
       - PULSAR_MEM=-Xms128m -Xmx128m -XX:MaxDirectMemorySize=56m
     command: >
       bash -c "bin/apply-config-from-env.py conf/zookeeper.conf && \
@@ -41,79 +43,79 @@ services:
       retries: 30
 
   # Initialize cluster metadata
-  pulsar-init:
+  green-pulsar-init:
     container_name: green-pulsar-init
-    hostname: pulsar-init
+    hostname: green-pulsar-init
     image: apachepulsar/pulsar:${PULSAR_VERSION}
     networks:
-      - green-pulsar
+      - extensible-load-manager_pulsar
     environment:
       - PULSAR_MEM=-Xms128m -Xmx128m -XX:MaxDirectMemorySize=56m
     command: >
       bin/pulsar initialize-cluster-metadata \
-               --cluster cluster-a \
-               --zookeeper zookeeper:2181 \
-               --configuration-store zookeeper:2181 \
-               --web-service-url http://broker-1:8080 \
-               --broker-service-url pulsar://broker-1:6650
+               --cluster cluster-green \
+               --zookeeper green-zookeeper:2181 \
+               --configuration-store green-zookeeper:2181 \
+               --web-service-url http://green-broker-1:8080 \
+               --broker-service-url pulsar://green-broker-1:6650
     depends_on:
-      zookeeper:
+      green-zookeeper:
         condition: service_healthy
 
   # Start bookie
-  bookie:
+  green-bookie:
     image: apachepulsar/pulsar:${PULSAR_VERSION}
     container_name: green-bookie
     restart: on-failure
     networks:
-      - green-pulsar
+      - extensible-load-manager_pulsar
     environment:
-      - clusterName=cluster-a
-      - zkServers=zookeeper:2181
-      - metadataServiceUri=metadata-store:zk:zookeeper:2181
-      - advertisedAddress=bookie
+      - clusterName=cluster-green
+      - zkServers=green-zookeeper:2181
+      - metadataServiceUri=metadata-store:zk:green-zookeeper:2181
+      - advertisedAddress=green-bookie
       - BOOKIE_MEM=-Xms128m -Xmx128m -XX:MaxDirectMemorySize=56m
     depends_on:
-      zookeeper:
+      green-zookeeper:
         condition: service_healthy
-      pulsar-init:
+      green-pulsar-init:
         condition: service_completed_successfully
     command: bash -c "bin/apply-config-from-env.py conf/bookkeeper.conf && exec bin/pulsar bookie"
 
-  proxy:
+  green-proxy:
     image: apachepulsar/pulsar:${PULSAR_VERSION}
     container_name: green-proxy
-    hostname: proxy
+    hostname: green-proxy
     restart: on-failure
     networks:
-      - green-pulsar
+      - extensible-load-manager_pulsar
     environment:
-      - metadataStoreUrl=zk:zookeeper:2181
-      - zookeeperServers=zookeeper:2181
-      - clusterName=cluster-a
+      - metadataStoreUrl=zk:green-zookeeper:2181
+      - zookeeperServers=green-zookeeper:2181
+      - clusterName=cluster-green
       - PULSAR_MEM=-Xms128m -Xmx128m -XX:MaxDirectMemorySize=56m
     ports:
       - "8081:8080"
       - "6651:6650"
     depends_on:
-      broker-1:
+      green-broker-1:
         condition: service_started
-      broker-2:
+      green-broker-2:
         condition: service_started
     command: bash -c "bin/apply-config-from-env.py conf/proxy.conf && exec bin/pulsar proxy"
 
   # Start broker 1
-  broker-1:
+  green-broker-1:
     image: apachepulsar/pulsar:${PULSAR_VERSION}
     container_name: green-broker-1
-    hostname: broker-1
+    hostname: green-broker-1
     restart: on-failure
     networks:
-      - green-pulsar
+      - extensible-load-manager_pulsar
     environment:
-      - metadataStoreUrl=zk:zookeeper:2181
-      - zookeeperServers=zookeeper:2181
-      - clusterName=cluster-a
+      - metadataStoreUrl=zk:green-zookeeper:2181
+      - zookeeperServers=green-zookeeper:2181
+      - clusterName=cluster-green
       - managedLedgerDefaultEnsembleSize=1
       - managedLedgerDefaultWriteQuorum=1
       - managedLedgerDefaultAckQuorum=1
@@ -129,24 +131,24 @@ services:
       - brokerServiceCompactionThresholdInBytes=1000000
       - PULSAR_PREFIX_defaultNumberOfNamespaceBundles=1
     depends_on:
-      zookeeper:
+      green-zookeeper:
         condition: service_healthy
-      bookie:
+      green-bookie:
         condition: service_started
     command: bash -c "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker"
 
   # Start broker 2
-  broker-2:
+  green-broker-2:
     image: apachepulsar/pulsar:${PULSAR_VERSION}
     container_name: green-broker-2
-    hostname: broker-2
+    hostname: green-broker-2
     restart: on-failure
     networks:
-      - green-pulsar
+      - extensible-load-manager_pulsar
     environment:
-      - metadataStoreUrl=zk:zookeeper:2181
-      - zookeeperServers=zookeeper:2181
-      - clusterName=cluster-a
+      - metadataStoreUrl=zk:green-zookeeper:2181
+      - zookeeperServers=green-zookeeper:2181
+      - clusterName=cluster-green
       - managedLedgerDefaultEnsembleSize=1
       - managedLedgerDefaultWriteQuorum=1
       - managedLedgerDefaultAckQuorum=1
@@ -162,8 +164,8 @@ services:
       - brokerServiceCompactionThresholdInBytes=1000000
       - PULSAR_PREFIX_defaultNumberOfNamespaceBundles=1
     depends_on:
-      zookeeper:
+      green-zookeeper:
         condition: service_healthy
-      bookie:
+      green-bookie:
         condition: service_started
     command: bash -c "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker"

pulsar/blue_green_migration_test.go

@@ -55,15 +55,15 @@ func (suite *BlueGreenMigrationTestSuite) TestTopicMigration() {
 
 		{
 			testCaseName:  "proxyConnection",
-			blueAdminURL:  "http://localhost:8080",
-			blueClientUrl: "pulsar://localhost:6650",
-			greenAdminURL: "http://localhost:8081",
+			blueAdminURL:  "http://proxy:8080",
+			blueClientUrl: "pulsar://proxy:6650",
+			greenAdminURL: "http://green-proxy:8080",
 			migrationBody: `
 						{
-							"serviceUrl": "http://localhost:8081",
-							"serviceUrlTls":"https://localhost:8085",
-							"brokerServiceUrl": "pulsar://localhost:6651",
-							"brokerServiceUrlTls": "pulsar+ssl://localhost:6655"
+							"serviceUrl": "http://green-proxy:8080",
+							"serviceUrlTls":"https://green-proxy:8081",
+							"brokerServiceUrl": "pulsar://green-proxy:6650",
+							"brokerServiceUrlTls": "pulsar+ssl://green-proxy:6651"
 						}
 					`,
 		},

scripts/run-ci-blue-green-cluster.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e -x
+
+go test -race -coverprofile=/tmp/coverage-blue_green_topic_migration -timeout=5m -tags extensible_load_manager -v -run TestBlueGreenMigrationTestSuite ./pulsar
+go tool cover -html=/tmp/coverage-blue_green_topic_migration -o coverage-blue_green_topic_migration.html