Add MySQL topology server implementation

.github/workflows/local_example.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: oracle-vm-16cpu-64gb-x86-64
     strategy:
       matrix:
-        topo: [etcd,zk2]
+        topo: [etcd,zk2,mysql]
 
     steps:
     - name: Skip CI

examples/common/env.sh

@@ -65,6 +65,10 @@ elif [ "${TOPO}" = "consul" ]; then
     #TODO: Remove underscore(_) flags in v25, replace them with dashed(-) notation
     TOPOLOGY_FLAGS="--topo-implementation consul --topo-global-server-address ${CONSUL_SERVER}:${CONSUL_HTTP_PORT} --topo-global-root vitess/global/"
     mkdir -p "${VTDATAROOT}/consul"
+elif [ "${TOPO}" = "mysql" ]; then
+    # The MySQL topo addr is a valid go DSN
+    MYSQL_TOPO_ADDR="topouser:topopass@tcp(127.0.0.1:3306)/topo"
+    TOPOLOGY_FLAGS="--topo_implementation mysql --topo_global_server_address $MYSQL_TOPO_ADDR --topo_global_root /vitess/global"
 else
     ETCD_SERVER="localhost:2379"
     #TODO: Remove underscore(_) flags in v25, replace them with dashed(-) notation

examples/common/scripts/mysql-up.sh

@@ -0,0 +1,155 @@
+#!/bin/bash
+
+# Copyright 2025 The Vitess Authors.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is an example script that uses MySQL on 3306 as a topo server.
+source "$(dirname "${BASH_SOURCE[0]:-$0}")/../env.sh"
+
+cell=${CELL:-'test'}
+
+echo "Setting up mysql topo.."
+
+# Check if MySQL is already running
+# Extract user, password, host and port from MYSQL_TOPO_ADDR (format: user:password@tcp(host:port)/database)
+MYSQL_USER_PASS=$(echo "$MYSQL_TOPO_ADDR" | sed 's/@.*//')
+MYSQL_USER=$(echo "$MYSQL_USER_PASS" | cut -d: -f1)
+MYSQL_PASS=$(echo "$MYSQL_USER_PASS" | cut -d: -f2)
+MYSQL_HOST_PORT=$(echo "$MYSQL_TOPO_ADDR" | sed 's/.*@tcp(\([^)]*\)).*/\1/')
+MYSQL_HOST=$(echo "$MYSQL_HOST_PORT" | cut -d: -f1)
+MYSQL_PORT=$(echo "$MYSQL_HOST_PORT" | cut -d: -f2)
+CREATEUSER="CREATE USER IF NOT EXISTS '${MYSQL_USER}'@'%' IDENTIFIED WITH mysql_native_password BY '${MYSQL_PASS}';"
+GRANTTOPO="GRANT ALL PRIVILEGES ON topo.* TO '${MYSQL_USER}'@'%'; GRANT REPLICATION SLAVE ON *.* TO '${MYSQL_USER}'@'%';"
+
+# For the MySQL topo implementation we expect to have a MySQL server running.
+# We first try to connect as the extracted user with the extracted password.
+if mysql -h "$MYSQL_HOST" -P "$MYSQL_PORT" -u "$MYSQL_USER" -p"$MYSQL_PASS" -e "SELECT 1" > /dev/null 2>&1; then
+    echo "MySQL is already running on $MYSQL_HOST:$MYSQL_PORT"
+elif mysql -h "$MYSQL_HOST" -P "$MYSQL_PORT" -u root -e "SELECT 1" > /dev/null 2>&1; then
+    # if that fails, we try to connect as root with no password *and* create the topo user and then execute the GRANT statement.
+    # if that fails, we bail and print an error message saying please create the topo user with this CREATE USER + GRANT statement.
+    echo "MySQL is running, but '${MYSQL_USER}' does not exist. Attempting to create '${MYSQL_USER}' and grant privileges..."
+    if ! mysql -u "$MYSQL_USER" -p"$MYSQL_PASS" -e "SELECT 1" 2>/dev/null; then
+        if ! mysql -u root -e "$CREATEUSER" 2>/dev/null; then
+            echo "Failed to create user '${MYSQL_USER}'@'%'."
+            echo "Please create the user manually with the following commands:"
+            echo "  mysql -u root -e \"$CREATEUSER\""
+            echo "  mysql -u root -e \"$GRANTTOPO\""
+            exit 1
+        fi
+        if ! mysql -u root -e "$GRANTTOPO" 2>/dev/null; then
+            echo "Failed to grant privileges to '${MYSQL_USER}'@'%'."
+            echo "Please run the following command manually:"
+            echo "  mysql -u root -e \"$GRANTTOPO\""
+            exit 1
+        fi
+    fi
+
+else
+    echo "MySQL is not running, attempting to start it..."
+
+    # Check if we're running in Docker (by checking for /.dockerenv file)
+    if [[ -f /.dockerenv ]]; then
+        echo "Detected Docker environment, starting MySQL..."
+
+        # Create MySQL data directory and tmp directory
+        mkdir -p "${VTDATAROOT}/mysql-topo"
+        mkdir -p "${VTDATAROOT}/tmp"
+
+        # Initialize MySQL data directory if it doesn't exist
+        if [[ ! -d "${VTDATAROOT}/mysql-topo/mysql" ]]; then
+            echo "Initializing MySQL data directory..."
+            mysqld --initialize-insecure --user=vitess --datadir="${VTDATAROOT}/mysql-topo" > "${VTDATAROOT}/tmp/mysql-init.out" 2>&1
+        fi
+
+        # Start MySQL server
+        echo "Starting MySQL server..."
+        mysqld --user=vitess \
+               --datadir="${VTDATAROOT}/mysql-topo" \
+               --socket="${VTDATAROOT}/mysql-topo/mysql.sock" \
+               --pid-file="${VTDATAROOT}/mysql-topo/mysql.pid" \
+               --port="$MYSQL_PORT" \
+               --bind-address="$MYSQL_HOST" \
+               --skip-networking=false \
+               --skip-mysqlx \
+               --gtid-mode=ON \
+               --enforce-gtid-consistency \
+               --log-bin \
+               --log-error="${VTDATAROOT}/tmp/mysql-topo.err" \
+               > "${VTDATAROOT}/tmp/mysql-topo.out" 2>&1 &
+
+        PID=$!
+        echo $PID > "${VTDATAROOT}/tmp/mysql-topo.pid"
+
+        # Wait for MySQL to start
+        echo "Waiting for MySQL to start..."
+        for i in {1..30}; do
+            if mysql -h "$MYSQL_HOST" -P "$MYSQL_PORT" -u root -e "SELECT 1" > /dev/null 2>&1; then
+                echo "MySQL started successfully"
+                break
+            fi
+            if [[ $i -eq 30 ]]; then
+                echo "Failed to start MySQL after 30 seconds"
+                echo "MySQL error log:"
+                cat "${VTDATAROOT}/tmp/mysql-topo.err" 2>/dev/null || echo "No error log found"
+                exit 1
+            fi
+            sleep 1
+        done
+
+        # Create the topo database
+        echo "Creating topo database..."
+        mysql -h "$MYSQL_HOST" -P "$MYSQL_PORT" -u root -e "CREATE DATABASE IF NOT EXISTS topo;" || {
+            echo "Failed to create topo database"
+            exit 1
+        }
+        # Creating the topo user and granting privileges
+        echo "Creating topo user and granting privileges..."
+        mysql -h "$MYSQL_HOST" -P "$MYSQL_PORT" -u root -e "$CREATEUSER" || {
+            echo "Failed to create user '${MYSQL_USER}'@'%'"
+            exit 1
+        }
+        mysql -h "$MYSQL_HOST" -P "$MYSQL_PORT" -u root -e "$GRANTTOPO" || {
+            echo "Failed to grant privileges to '${MYSQL_USER}'@'%'"
+            exit 1
+        }
+
+    else
+        echo "Error: MySQL is not running and this script can only auto-start MySQL in Docker environments."
+        echo "Please start MySQL manually before calling mysql-up.sh"
+        echo "You can start MySQL with a command like:"
+        echo "  sudo systemctl start mysql"
+        echo "  # or"
+        echo "  sudo service mysql start"
+        echo "  # or start it manually with mysqld"
+        echo ""
+        echo "If using Docker, you can start a MySQL container with:"
+        echo "  docker run -d --name mysql-topo -p ${MYSQL_PORT}:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=yes mysql:8.0 --gtid-mode=ON --enforce-gtid-consistency --log-bin"
+        exit 1
+    fi
+fi
+
+mysql -h "$MYSQL_HOST" -P "$MYSQL_PORT" -u "$MYSQL_USER" -p"$MYSQL_PASS" -e "CREATE DATABASE IF NOT EXISTS topo" || true
+
+# And also add the CellInfo description for the cell.
+# If the node already exists, it's fine, means we used existing data.
+echo "add ${cell} CellInfo"
+set +e
+command vtctldclient --server internal --topo-implementation mysql --topo-global-server-address $MYSQL_TOPO_ADDR AddCellInfo \
+  --root "/vitess/${cell}" \
+  --server-address "$MYSQL_TOPO_ADDR" \
+  "${cell}"
+set -e
+
+echo "mysql topo set!"

examples/local/101_initial_cluster.sh

@@ -33,6 +33,8 @@ if [ "${TOPO}" = "zk2" ]; then
 	CELL=zone1 ../common/scripts/zk-up.sh
 elif [ "${TOPO}" = "consul" ]; then
 	CELL=zone1 ../common/scripts/consul-up.sh
+elif [ "${TOPO}" = "mysql" ]; then
+	CELL=zone1 ../common/scripts/mysql-up.sh
 else
 	CELL=zone1 ../common/scripts/etcd-up.sh
 fi

examples/local/501_teardown.sh

@@ -51,6 +51,11 @@ if [ "${TOPO}" = "zk2" ]; then
 	CELL=zone1 ../common/scripts/zk-down.sh
 elif [ "${TOPO}" = "consul" ]; then
 	CELL=zone1 ../common/scripts/consul-down.sh
+elif [ "${TOPO}" = "mysql" ]; then
+	# MySQL doesn't have a down script, since we externally manage it.
+	# But just below in the cleanup we will empty the TOPO schema
+	# so that it is ready for the next run.
+	echo "The MySQL topo does not have a down script"
 else
 	CELL=zone1 ../common/scripts/etcd-down.sh
 fi
@@ -67,6 +72,23 @@ if [ -n "$VTDATAROOT" ]; then
 
 	# shellcheck disable=SC2086
 	rm -r ${VTDATAROOT:?}/*
+
+	# For the MySQL topo specifically, we do not store data in VTDATAROOT,
+	# it is externally managed instead. If we don't wipe it, the next run
+	# will fail because the MySQL topo will not be empty, and already know about
+	# various tables causing an error:
+	# ERROR: Cannot determine primary tablet for keyspace/shard commerce/0
+	if [ "${TOPO}" = "mysql" ]; then
+		echo "Cleaning up MySQL topology database..."
+		# Extract user, password, host and port from MYSQL_TOPO_ADDR (format: user:password@tcp(host:port)/database)
+		MYSQL_USER_PASS=$(echo "$MYSQL_TOPO_ADDR" | sed 's/@.*//')
+		MYSQL_USER=$(echo "$MYSQL_USER_PASS" | cut -d: -f1)
+		MYSQL_PASS=$(echo "$MYSQL_USER_PASS" | cut -d: -f2)
+		MYSQL_HOST_PORT=$(echo "$MYSQL_TOPO_ADDR" | sed 's/.*@tcp(\([^)]*\)).*/\1/')
+		MYSQL_HOST=$(echo "$MYSQL_HOST_PORT" | cut -d: -f1)
+		MYSQL_PORT=$(echo "$MYSQL_HOST_PORT" | cut -d: -f2)
+		mysql -h "$MYSQL_HOST" -P "$MYSQL_PORT" -u "$MYSQL_USER" -p"$MYSQL_PASS" -e "DROP DATABASE IF EXISTS topo"
+	fi
 fi
 
 disown -a

go/cmd/topo2topo/cli/plugin_mysqltopo.go

@@ -0,0 +1,23 @@
+/*
+Copyright 2025 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cli
+
+// This plugin imports mysqltopo to register the mysql implementation of TopoServer.
+
+import (
+	_ "vitess.io/vitess/go/vt/topo/mysqltopo"
+)

go/cmd/vtctld/cli/plugin_mysqltopo.go

@@ -0,0 +1,23 @@
+/*
+Copyright 2025 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cli
+
+// This plugin imports mysqltopo to register the mysql implementation of TopoServer.
+
+import (
+	_ "vitess.io/vitess/go/vt/topo/mysqltopo"
+)

go/cmd/vtctldclient/command/root.go

@@ -53,6 +53,7 @@ import (
 	// These imports register the topo factories to use when --server=internal.
 	_ "vitess.io/vitess/go/vt/topo/consultopo"
 	_ "vitess.io/vitess/go/vt/topo/etcd2topo"
+	_ "vitess.io/vitess/go/vt/topo/mysqltopo"
 	_ "vitess.io/vitess/go/vt/topo/zk2topo"
 )
 

go/cmd/vtgate/cli/plugin_mysqltopo.go

@@ -0,0 +1,23 @@
+/*
+Copyright 2025 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cli
+
+// This plugin imports mysqltopo to register the mysql implementation of TopoServer.
+
+import (
+	_ "vitess.io/vitess/go/vt/topo/mysqltopo"
+)

go/cmd/vtorc/cli/plugin_mysqltopo.go

@@ -0,0 +1,23 @@
+/*
+Copyright 2025 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cli
+
+// This plugin imports mysqltopo to register the mysql implementation of TopoServer.
+
+import (
+	_ "vitess.io/vitess/go/vt/topo/mysqltopo"
+)

go/cmd/vttablet/cli/plugin_mysqltopo.go

@@ -0,0 +1,23 @@
+/*
+Copyright 2025 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cli
+
+// This plugin imports mysqltopo to register the mysql implementation of TopoServer.
+
+import (
+	_ "vitess.io/vitess/go/vt/topo/mysqltopo"
+)

go/flags/endtoend/vtctld.txt

@@ -167,6 +167,8 @@ Flags:
       --topo-global-root string                                          the path of the global topology data in the global topology server
       --topo-global-server-address string                                the address of the global topology server
       --topo-implementation string                                       the topology implementation to use
+      --topo-mysql-election-ttl int                                      election TTL in seconds for MySQL topo (default 30)
+      --topo-mysql-lock-ttl int                                          lock TTL in seconds for MySQL topo (default 30)
       --topo-read-concurrency int                                        Maximum concurrency of topo reads per global or local cell. (default 32)
       --topo-zk-auth-file string                                         auth to use when connecting to the zk topo server, file contents should be <scheme>:<auth>, e.g., digest:user:pass
       --topo-zk-base-timeout duration                                    zk base timeout (see zk.Connect) (default 30s)

go/flags/endtoend/vtgate.txt

@@ -228,6 +228,8 @@ Flags:
       --topo-global-root string                                          the path of the global topology data in the global topology server
       --topo-global-server-address string                                the address of the global topology server
       --topo-implementation string                                       the topology implementation to use
+      --topo-mysql-election-ttl int                                      election TTL in seconds for MySQL topo (default 30)
+      --topo-mysql-lock-ttl int                                          lock TTL in seconds for MySQL topo (default 30)
       --topo-read-concurrency int                                        Maximum concurrency of topo reads per global or local cell. (default 32)
       --topo-zk-auth-file string                                         auth to use when connecting to the zk topo server, file contents should be <scheme>:<auth>, e.g., digest:user:pass
       --topo-zk-base-timeout duration                                    zk base timeout (see zk.Connect) (default 30s)

go/flags/endtoend/vtorc.txt

@@ -103,6 +103,8 @@ Flags:
       --topo-global-server-address string                           the address of the global topology server
       --topo-implementation string                                  the topology implementation to use
       --topo-information-refresh-duration duration                  Timer duration on which VTOrc refreshes the keyspace and vttablet records from the topology server (default 15s)
+      --topo-mysql-election-ttl int                                 election TTL in seconds for MySQL topo (default 30)
+      --topo-mysql-lock-ttl int                                     lock TTL in seconds for MySQL topo (default 30)
       --topo-read-concurrency int                                   Maximum concurrency of topo reads per global or local cell. (default 32)
       --topo-zk-auth-file string                                    auth to use when connecting to the zk topo server, file contents should be <scheme>:<auth>, e.g., digest:user:pass
       --topo-zk-base-timeout duration                               zk base timeout (see zk.Connect) (default 30s)

go/flags/endtoend/vttablet.txt

@@ -379,6 +379,8 @@ Flags:
       --topo-global-root string                                          the path of the global topology data in the global topology server
       --topo-global-server-address string                                the address of the global topology server
       --topo-implementation string                                       the topology implementation to use
+      --topo-mysql-election-ttl int                                      election TTL in seconds for MySQL topo (default 30)
+      --topo-mysql-lock-ttl int                                          lock TTL in seconds for MySQL topo (default 30)
       --topo-read-concurrency int                                        Maximum concurrency of topo reads per global or local cell. (default 32)
       --topo-zk-auth-file string                                         auth to use when connecting to the zk topo server, file contents should be <scheme>:<auth>, e.g., digest:user:pass
       --topo-zk-base-timeout duration                                    zk base timeout (see zk.Connect) (default 30s)