DISTMYSQL-422: Intermediate master failover does not work for 8.0.33

https://perconadev.atlassian.net/browse/DISTMYSQL-422

Problem/Cause:
8.0.33 changed messages displayed by SHOW REPLICA STATUS substituting
master/slave with source/replica. Orchestrator's failover intermediate
master failover mechanism relies on these messages.

Solution:
Adjusted messages parsing.
Introduced new tests.

Author: kamil-holubicki

go/inst/analysis_dao.go

@@ -24,7 +24,7 @@ import (
 	"github.com/openark/orchestrator/go/config"
 	"github.com/openark/orchestrator/go/db"
 	"github.com/openark/orchestrator/go/process"
-	"github.com/openark/orchestrator/go/raft"
+	orcraft "github.com/openark/orchestrator/go/raft"
 	"github.com/openark/orchestrator/go/util"
 
 	"github.com/openark/golib/log"
@@ -73,7 +73,10 @@ func GetReplicationAnalysis(clusterName string, hints *ReplicationAnalysisHints)
 						SUM(
 							replica_instance.last_checked <= replica_instance.last_seen
 							AND replica_instance.slave_io_running = 0
-							AND replica_instance.last_io_error like '%error %connecting to master%'
+							AND (
+								replica_instance.last_io_error like '%error %connecting to master%'
+								OR replica_instance.last_io_error like '%error %connecting to source%'
+								)
 							AND replica_instance.slave_sql_running = 1
 						),
 						0
@@ -109,7 +112,10 @@ func GetReplicationAnalysis(clusterName string, hints *ReplicationAnalysisHints)
 					MIN(
 						master_instance.slave_sql_running = 1
 						AND master_instance.slave_io_running = 0
-						AND master_instance.last_io_error like '%error %connecting to master%'
+						AND (
+							master_instance.last_io_error like '%error %connecting to master%'
+							OR master_instance.last_io_error like '%error %connecting to source%'
+							)
 					)
 					/* AS is_failing_to_connect_to_master */
 				)
@@ -210,7 +216,10 @@ func GetReplicationAnalysis(clusterName string, hints *ReplicationAnalysisHints)
 			SUM(
 				replica_instance.last_checked <= replica_instance.last_seen
 				AND replica_instance.slave_io_running = 0
-				AND replica_instance.last_io_error like '%%error %%connecting to master%%'
+				AND (
+					replica_instance.last_io_error like '%%error %%connecting to master%%'
+				    OR replica_instance.last_io_error like '%%error %%connecting to source%%'
+				    )
 				AND replica_instance.slave_sql_running = 1
 			),
 			0
@@ -226,7 +235,10 @@ func GetReplicationAnalysis(clusterName string, hints *ReplicationAnalysisHints)
 		MIN(
 			master_instance.slave_sql_running = 1
 			AND master_instance.slave_io_running = 0
-			AND master_instance.last_io_error like '%%error %%connecting to master%%'
+			AND (
+				master_instance.last_io_error like '%%error %%connecting to master%%'
+				OR master_instance.last_io_error like '%%error %%connecting to source%%'
+				)
 		) AS is_failing_to_connect_to_master,
 		MIN(
 			master_downtime.downtime_active is not null

run/test-integration.sh

@@ -7,7 +7,7 @@ export CI_ENV_REPO=
 export CI_ENV_BRANCH=
 
 # Configure test run parameters
-export TARBALL_URL=https://downloads.percona.com/downloads/Percona-Server-8.0/Percona-Server-8.0.26-16/binary/tarball/Percona-Server-8.0.26-16-Linux.x86_64.glibc2.12-minimal.tar.gz
+export TARBALL_URL=https://downloads.percona.com/downloads/Percona-Server-8.0/Percona-Server-8.0.36-28/binary/tarball/Percona-Server-8.0.36-28-Linux.x86_64.glibc2.17-minimal.tar.gz
 export RUN_TESTS=YES
 export ALLOW_TESTS_FAILURES=YES
 

run/test-system.sh

@@ -7,7 +7,7 @@ export CI_ENV_REPO=
 export CI_ENV_BRANCH=
 
 # Configure test run parameters
-export TARBALL_URL=https://downloads.percona.com/downloads/Percona-Server-8.0/Percona-Server-8.0.30-22/binary/tarball/Percona-Server-8.0.30-22-Linux.x86_64.glibc2.17-minimal.tar.gz
+export TARBALL_URL=https://downloads.percona.com/downloads/Percona-Server-8.0/Percona-Server-8.0.36-28/binary/tarball/Percona-Server-8.0.36-28-Linux.x86_64.glibc2.17-minimal.tar.gz
 export RUN_TESTS=YES
 export ALLOW_TESTS_FAILURES=YES
 

tests/integration/analysis-dead-intermediate-master-with-single-replica-failing-to-connect-8033/create.sql

@@ -0,0 +1,3 @@
+-- 22295 replicates from 22294
+UPDATE database_instance SET slave_io_running=0, last_io_error='error connecting to source' where master_port=22294;
+UPDATE database_instance SET last_seen=last_checked - interval 1 minute where port=22294;

tests/integration/analysis-dead-intermediate-master-with-single-replica-failing-to-connect-8033/expect_output

@@ -0,0 +1 @@
+testhost:22294 (cluster testhost:22293): DeadIntermediateMasterWithSingleReplicaFailingToConnect

tests/integration/analysis-dead-intermediate-master-with-single-replica-failing-to-connect-8033/extra_args

@@ -0,0 +1 @@
+-c replication-analysis

tests/integration/analysis-first-tier-replica-failing-to-connect-8033/create.sql

@@ -0,0 +1 @@
+UPDATE database_instance SET slave_io_running=0, last_io_error='error connecting to source' where port=22294;

tests/integration/analysis-first-tier-replica-failing-to-connect-8033/expect_output

@@ -0,0 +1 @@
+testhost:22294 (cluster testhost:22293): FirstTierReplicaFailingToConnectToMaster

tests/integration/analysis-first-tier-replica-failing-to-connect-8033/extra_args

@@ -0,0 +1 @@
+-c replication-analysis

tests/integration/analysis-unreachable-master-partial-success-broken-replica-8033/create.sql

@@ -0,0 +1,3 @@
+UPDATE database_instance SET last_seen=last_checked - interval 1 minute where port=22293;
+UPDATE database_instance SET last_check_partial_success = 1 where port=22293;
+UPDATE database_instance SET slave_io_running=0, last_io_error='error connecting to source' where port=22294;