test: fix flaky test

Helper method that performs initial assigning of master/replica roles
and is widely used in ConnectionPool tests was adjusted to wait for
the roles to be applied successfully.
Prior to this patch it doesn't, so sometimes subsequent test code might
work unexpectedly (the problem was caught with
TestConnectionHandlerOpenUpdateClose)

Closes #452

Author: elhimov

.github/workflows/testing.yml

@@ -273,6 +273,9 @@ jobs:
           cd "${SRCDIR}"
           make deps
 
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
+
       - name: Run regression tests
         run: |
           cd "${SRCDIR}"

test_helpers/pool_helper.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"reflect"
+	"sync"
 	"time"
 
 	"github.com/tarantool/go-tarantool/v2"
@@ -195,28 +196,83 @@ func SetInstanceRO(ctx context.Context, instance pool.Instance, isReplica bool)
 
 	req := tarantool.NewCallRequest("box.cfg").
 		Args([]interface{}{map[string]bool{"read_only": isReplica}})
-	if _, err := conn.Do(req).Get(); err != nil {
-		return err
+	if resp, err := conn.Do(req).Get(); err != nil {
+		// return err
+		return fmt.Errorf("%w: resp: %v", err, resp)
 	}
 
 	return nil
 }
 
+type initPoolHandler struct {
+	roles      map[string]pool.Role
+	rolesMutex sync.Mutex
+}
+
+func (h *initPoolHandler) Discovered(name string, conn *tarantool.Connection,
+	role pool.Role) error {
+	h.rolesMutex.Lock()
+	h.roles[name] = role
+	h.rolesMutex.Unlock()
+	return nil
+}
+
+func (h *initPoolHandler) Deactivated(name string, conn *tarantool.Connection,
+	role pool.Role) error {
+	h.rolesMutex.Lock()
+	delete(h.roles, name)
+	h.rolesMutex.Unlock()
+	return nil
+}
+
 func SetClusterRO(instances []pool.Instance, roles []bool) error {
 	if len(instances) != len(roles) {
 		return fmt.Errorf("number of instances should be equal to number of roles")
 	}
 
+	ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
+
 	for i, instance := range instances {
-		ctx, cancel := GetConnectContext()
-		err := SetInstanceRO(ctx, instance, roles[i])
+		err := SetInstanceRO(ctx, instance.Dialer, instance.Opts, roles[i])
 		cancel()
 		if err != nil {
 			return err
 		}
 	}
 
-	return nil
+	// if ctx.Done() {
+	// 	return nil, ctx.Err()
+	// }
+
+	h := &initPoolHandler{}
+	initPoolOpts := pool.Opts{
+		CheckTimeout:      100 * time.Microsecond,
+		ConnectionHandler: h,
+	}
+	pool, err := pool.ConnectWithOpts(ctx, instances, initPoolOpts)
+	if err != nil {
+		return err
+	}
+	defer pool.Close()
+
+	// Prepare roles map that
+	expectedRoles := make(map[string]bool, len(roles))
+	for i, role := range roles {
+		expectedRoles[instances[i].Name] = role
+	}
+
+	// Wait for the roles to be applied.
+	for i := 0; i < 100; i++ {
+		h.rolesMutex.Lock()
+		if reflect.DeepEqual(h.roles, expectedRoles) {
+			h.rolesMutex.Unlock()
+			return nil
+		}
+		h.rolesMutex.Unlock()
+
+		time.Sleep(100 * time.Millisecond)
+	}
+	return fmt.Errorf("Failed to wait roles are applied")
 }
 
 func StartTarantoolInstances(instsOpts []StartOpts) ([]*TarantoolInstance, error) {
@@ -242,5 +298,5 @@ func StopTarantoolInstances(instances []*TarantoolInstance) {
 }
 
 func GetPoolConnectContext() (context.Context, context.CancelFunc) {
-	return context.WithTimeout(context.Background(), 500*time.Millisecond)
+	return context.WithTimeout(context.Background(), 1000*time.Millisecond)
 }