DISTMYSQL-243 - Orchestrator uses inefficient subquery in REPLACE to update cluster aliases

Problem:
--------
Orchestrator has backend table cluster_alias to store aliases.
At certain intervals, orchestrator will update the aliases or insert new host aliases.
To do this, it uses the below query in UpdateClusterAliases():
```
			replace into
					cluster_alias (alias, cluster_name, last_registered)
				select
				    suggested_cluster_alias,
						cluster_name,
						now()
					from
				    database_instance
				    left join database_instance_downtime using (hostname, port)
				  where
				    suggested_cluster_alias!=''
						/* exclude newly demoted, downtimed masters */
						and ifnull(
								database_instance_downtime.downtime_active = 1
								and database_instance_downtime.end_timestamp > now()
								and database_instance_downtime.reason = ?
							, 0) = 0
					order by
						ifnull(last_checked <= last_seen, 0) asc,
						read_only desc,
						num_slave_hosts asc

```

The problem with the select query is, it will generate the duplicate records. Note that the "order" of records is important
to determine the active nodes and the alias should only be updated to active nodes.

REPLACE does this operation by doing DELETE+INSERT. REPLACE repeatedly does the same work for all the duplicated records.

This creates un-necessary work for two sub systems in InnoDB
1. Purge
2. SELECTS (ReadView)

All those delete marked records create stress on Purge.
SELECTs, when they have to build a older version of record, they have to build a long chain of old version records (using undo log).
So the un-necessary REPLACE work will create a long chain of records to be built.

Fix:
----
Since the order is important, we cannot use "GROUP BY" to remove duplicates.
Use Insert On Duplicate Key Update (IODKU) instead of REPLACE as it generates less work for InnoDB

IODKU:
------

```
INSERT INTO cluster_alias(alias, cluster_name, last_registered)
SELECT di.suggested_cluster_alias,
       di.cluster_name,
       now()
FROM   database_instance di
       LEFT JOIN database_instance_downtime did USING (hostname, port)
WHERE  di.suggested_cluster_alias != ''
       AND Ifnull(did.downtime_active = 1
                  AND did.end_timestamp > Now()
                  AND did.reason = 'lost-in-recovery', 0)
           = 0
ORDER  BY Ifnull(di.last_checked <= di.last_seen, 0) ASC,
          di.read_only DESC,
          di.num_slave_hosts ASC
ON DUPLICATE KEY UPDATE
alias = di.suggested_cluster_alias,
cluster_name = di.cluster_name,
last_registered = now();
```

Sample Performance numbers:
----------------------------
REPLACE:

Query OK, 14880 rows affected (16.93 sec)
Records: 7440  Duplicates: 7440  Warnings: 0

IODKU:

Query OK, 468 rows affected (6.17 sec)
Records: 7440  Duplicates: 234  Warnings: 0

Observe the number of records affected,number of duplicates affected and the time. IODKU is way better.

Author: satya-bodapati

go/inst/cluster_alias_dao.go

@@ -25,6 +25,10 @@ import (
 	"github.com/openark/orchestrator/go/db"
 )
 
+func IsSQLite() bool {
+	return config.Config.IsSQLite()
+}
+
 // ReadClusterNameByAlias
 func ReadClusterNameByAlias(alias string) (clusterName string, err error) {
 	query := `
@@ -114,29 +118,62 @@ func writeClusterAliasManualOverride(clusterName string, alias string) error {
 // gained from database_instance
 func UpdateClusterAliases() error {
 	writeFunc := func() error {
-		_, err := db.ExecOrchestrator(`
-			replace into
-					cluster_alias (alias, cluster_name, last_registered)
-				select
-				    suggested_cluster_alias,
-						cluster_name,
-						now()
-					from
-				    database_instance
-				    left join database_instance_downtime using (hostname, port)
-				  where
-				    suggested_cluster_alias!=''
-						/* exclude newly demoted, downtimed masters */
-						and ifnull(
-								database_instance_downtime.downtime_active = 1
-								and database_instance_downtime.end_timestamp > now()
-								and database_instance_downtime.reason = ?
-							, 0) = 0
-					order by
-						ifnull(last_checked <= last_seen, 0) asc,
-						read_only desc,
-						num_slave_hosts asc
-			`, DowntimeLostInRecoveryMessage)
+		var err error
+		if IsSQLite() {
+			// Sql lite backend
+			_, err = db.ExecOrchestrator(`
+				replace into
+						cluster_alias (alias, cluster_name, last_registered)
+					select
+					    suggested_cluster_alias,
+							cluster_name,
+							now()
+						from
+					    database_instance
+					    left join database_instance_downtime using (hostname, port)
+					  where
+					    suggested_cluster_alias!=''
+							/* exclude newly demoted, downtimed masters */
+							and ifnull(
+									database_instance_downtime.downtime_active = 1
+									and database_instance_downtime.end_timestamp > now()
+									and database_instance_downtime.reason = ?
+								, 0) = 0
+						order by
+							ifnull(last_checked <= last_seen, 0) asc,
+							read_only desc,
+							num_slave_hosts asc
+				`, DowntimeLostInRecoveryMessage)
+		} else {
+			// MySQL backend (Orchestrator supports only SQLite and MySQL backends)
+			// INSERT ON DUPLICATE KEY UPDATE is more performant than REPLACE in MySQL
+			_, err = db.ExecOrchestrator(`
+				INSERT INTO cluster_alias
+				(
+					alias,
+					cluster_name,
+					last_registered
+				)
+				SELECT    di.suggested_cluster_alias,
+					  di.cluster_name,
+					  now()
+				FROM      database_instance di
+				LEFT JOIN database_instance_downtime did
+				USING     (hostname, port)
+				WHERE     di.suggested_cluster_alias != ''
+				/* exclude newly demoted, downtimed masters */
+				AND       Ifnull(did.downtime_active = 1
+				AND       did.end_timestamp > Now()
+				AND       did.reason = ?, 0) = 0
+				ORDER BY  Ifnull(di.last_checked <= di.last_seen, 0) ASC,
+					  di.read_only DESC,
+					  di.num_slave_hosts ASC
+				ON DUPLICATE KEY
+				UPDATE alias = di.suggested_cluster_alias,
+				cluster_name = di.cluster_name,
+				last_registered = now()
+				`, DowntimeLostInRecoveryMessage)
+		}
 		return log.Errore(err)
 	}
 	if err := ExecDBWriteFunc(writeFunc); err != nil {