Add "DNS Check Interval" option. Fixes #1201

Author: bgrainger

docs/content/connection-options.md

@@ -232,6 +232,15 @@ Connection pooling is enabled by default. These options are used to configure it
     <td>100</td>
     <td>The maximum number of connections allowed in the pool.</td>
   </tr>
+  <tr id="DnsCheckInterval">
+    <td>DNS Check Interval, DnsCheckInterval</td>
+    <td>0</td>
+    <td>The number of seconds between checks for DNS changes, or 0 to disable periodic checks.
+    If the periodic check determines that one of the <code>Server</code> hostnames resolves to a different IP address, the pool will be cleared.
+    This is useful in HA scenarios where failover is accomplished by changing the IP address to which a hostname resolves.
+    Existing connections in the pool may have valid TCP connections to a server that is no longer responding or has been marked readonly;
+    clearing the pool (when DNS changes) forces all these existing connections to be reestablished.</td>
+  </tr>
 </table>
 
 ## Other Options

src/MySqlConnector/Core/ConnectionPool.cs

@@ -1,12 +1,15 @@
 using System.Collections.Concurrent;
+using System.Net;
 using System.Security.Authentication;
 using MySqlConnector.Logging;
 using MySqlConnector.Protocol.Serialization;
 using MySqlConnector.Utilities;
 
 namespace MySqlConnector.Core;
 
+#pragma warning disable CA1001 // Types that own disposable fields should be disposable
 internal sealed class ConnectionPool
+#pragma warning restore CA1001 // Types that own disposable fields should be disposable
 {
 	public int Id { get; }
 
@@ -438,6 +441,7 @@ private async ValueTask<ServerSession> ConnectSessionAsync(MySqlConnection conne
 		var connectionSettings = new ConnectionSettings(connectionStringBuilder);
 		var pool = new ConnectionPool(connectionSettings);
 		pool.StartReaperTask();
+		pool.StartDnsCheckTimer();
 		return pool;
 	}
 
@@ -485,6 +489,7 @@ private async ValueTask<ServerSession> ConnectSessionAsync(MySqlConnection conne
 		{
 			s_mruCache = new(connectionString, pool);
 			pool.StartReaperTask();
+			pool.StartDnsCheckTimer();
 
 			// if we won the race to create the new pool, also store it under the original connection string
 			if (connectionString != normalizedConnectionString)
@@ -569,6 +574,92 @@ private void StartReaperTask()
 		}
 	}
 
+	private void StartDnsCheckTimer()
+	{
+		if (ConnectionSettings.ConnectionProtocol != MySqlConnectionProtocol.Tcp || ConnectionSettings.DnsCheckInterval <= 0)
+			return;
+
+		var hostNames = ConnectionSettings.HostNames!;
+		var hostAddresses = new IPAddress[hostNames.Count][];
+
+#if NET6_0_OR_GREATER
+		m_dnsCheckTimer = new PeriodicTimer(TimeSpan.FromSeconds(ConnectionSettings.DnsCheckInterval));
+		_ = RunTimer();
+
+		async Task RunTimer()
+		{
+			while (await m_dnsCheckTimer.WaitForNextTickAsync().ConfigureAwait(false))
+			{
+				Log.Trace("Pool{0} checking for DNS changes", m_logArguments);
+				var hostNamesChanged = false;
+				for (var hostNameIndex = 0; hostNameIndex < hostNames.Count; hostNameIndex++)
+				{
+					try
+					{
+						var ipAddresses = await Dns.GetHostAddressesAsync(hostNames[hostNameIndex]).ConfigureAwait(false);
+						if (hostAddresses[hostNameIndex] is null)
+						{
+							hostAddresses[hostNameIndex] = ipAddresses;
+						}
+						else if (hostAddresses[hostNameIndex].Except(ipAddresses).Any())
+						{
+							Log.Debug("Pool{0} detected DNS change for HostName '{1}': {2} to {3}", m_logArguments[0], hostNames[hostNameIndex], string.Join<IPAddress>(',', hostAddresses[hostNameIndex]), string.Join<IPAddress>(',', ipAddresses));
+							hostAddresses[hostNameIndex] = ipAddresses;
+							hostNamesChanged = true;
+						}
+					}
+					catch (Exception ex)
+					{
+						// do nothing; we'll try again later
+						Log.Debug("Pool{0} DNS check failed; ignoring HostName '{1}': {2}", m_logArguments[0], hostNames[hostNameIndex], ex.Message);
+					}
+				}
+				if (hostNamesChanged)
+				{
+					Log.Info("Pool{0} clearing pool due to DNS changes", m_logArguments);
+					await ClearAsync(IOBehavior.Asynchronous, CancellationToken.None).ConfigureAwait(false);
+				}
+			}
+		}
+#else
+		var interval = Math.Min(int.MaxValue / 1000, ConnectionSettings.DnsCheckInterval) * 1000;
+		m_dnsCheckTimer = new Timer(t =>
+		{
+			Log.Trace("Pool{0} checking for DNS changes", m_logArguments);
+			var hostNamesChanged = false;
+			for (var hostNameIndex = 0; hostNameIndex < hostNames.Count; hostNameIndex++)
+			{
+				try
+				{
+					var ipAddresses = Dns.GetHostAddresses(hostNames[hostNameIndex]);
+					if (hostAddresses[hostNameIndex] is null)
+					{
+						hostAddresses[hostNameIndex] = ipAddresses;
+					}
+					else if (hostAddresses[hostNameIndex].Except(ipAddresses).Any())
+					{
+						Log.Debug("Pool{0} detected DNS change for HostName '{1}': {2} to {3}", m_logArguments[0], hostNames[hostNameIndex], string.Join<IPAddress>(",", hostAddresses[hostNameIndex]), string.Join<IPAddress>(",", ipAddresses));
+						hostAddresses[hostNameIndex] = ipAddresses;
+						hostNamesChanged = true;
+					}
+				}
+				catch (Exception ex)
+				{
+					// do nothing; we'll try again later
+					Log.Debug("Pool{0} DNS check failed; ignoring HostName '{1}': {2}", m_logArguments[0], hostNames[hostNameIndex], ex.Message);
+				}
+			}
+			if (hostNamesChanged)
+			{
+				Log.Info("Pool{0} clearing pool due to DNS changes", m_logArguments);
+				ClearAsync(IOBehavior.Synchronous, CancellationToken.None).GetAwaiter().GetResult();
+			}
+			((Timer) t!).Change(interval, -1);
+		});
+		m_dnsCheckTimer.Change(interval, -1);
+#endif
+	}
+
 	private void AdjustHostConnectionCount(ServerSession session, int delta)
 	{
 		if (m_hostSessions is not null)
@@ -630,4 +721,9 @@ private static void OnAppDomainShutDown(object? sender, EventArgs e) =>
 	private uint m_lastRecoveryTime;
 	private int m_lastSessionId;
 	private Dictionary<string, CachedProcedure?>? m_procedureCache;
+#if NET6_0_OR_GREATER
+	private PeriodicTimer? m_dnsCheckTimer;
+#else
+	private Timer? m_dnsCheckTimer;
+#endif
 }

src/MySqlConnector/Core/ConnectionSettings.cs

@@ -119,6 +119,7 @@ public ConnectionSettings(MySqlConnectionStringBuilder csb)
 			throw new MySqlException("MaximumPoolSize must be greater than or equal to MinimumPoolSize");
 		MinimumPoolSize = ToSigned(csb.MinimumPoolSize);
 		MaximumPoolSize = ToSigned(csb.MaximumPoolSize);
+		DnsCheckInterval = ToSigned(csb.DnsCheckInterval);
 
 		// Other Options
 		AllowLoadLocalInfile = csb.AllowLoadLocalInfile;
@@ -213,6 +214,7 @@ private static MySqlGuidFormat GetEffectiveGuidFormat(MySqlGuidFormat guidFormat
 	public int ConnectionIdleTimeout { get; }
 	public int MinimumPoolSize { get; }
 	public int MaximumPoolSize { get; }
+	public int DnsCheckInterval { get; }
 
 	// Other Options
 	public bool AllowLoadLocalInfile { get; }
@@ -299,6 +301,7 @@ private ConnectionSettings(ConnectionSettings other, string host, int port, stri
 		ConnectionIdleTimeout = other.ConnectionIdleTimeout;
 		MinimumPoolSize = other.MinimumPoolSize;
 		MaximumPoolSize = other.MaximumPoolSize;
+		DnsCheckInterval = other.DnsCheckInterval;
 
 		AllowLoadLocalInfile = other.AllowLoadLocalInfile;
 		AllowPublicKeyRetrieval = other.AllowPublicKeyRetrieval;

src/MySqlConnector/MySqlConnectionStringBuilder.cs

@@ -404,6 +404,19 @@ public uint MaximumPoolSize
 		set => MySqlConnectionStringOption.MaximumPoolSize.SetValue(this, value);
 	}
 
+	/// <summary>
+	/// The number of seconds between checks for DNS changes, or 0 to disable periodic checks.
+	/// </summary>
+	[Category("Pooling")]
+	[DefaultValue(0u)]
+	[Description("The number of seconds between checks for DNS changes.")]
+	[DisplayName("DNS Check Interval")]
+	public uint DnsCheckInterval
+	{
+		get => MySqlConnectionStringOption.DnsCheckInterval.GetValue(this);
+		set => MySqlConnectionStringOption.DnsCheckInterval.SetValue(this, value);
+	}
+
 	// Other Options
 
 	/// <summary>
@@ -915,6 +928,7 @@ internal abstract partial class MySqlConnectionStringOption
 	public static readonly MySqlConnectionStringValueOption<uint> ConnectionIdleTimeout;
 	public static readonly MySqlConnectionStringValueOption<uint> MinimumPoolSize;
 	public static readonly MySqlConnectionStringValueOption<uint> MaximumPoolSize;
+	public static readonly MySqlConnectionStringValueOption<uint> DnsCheckInterval;
 
 	// Other Options
 	public static readonly MySqlConnectionStringValueOption<bool> AllowLoadLocalInfile;
@@ -1118,6 +1132,10 @@ static MySqlConnectionStringOption()
 			keys: new[] { "Maximum Pool Size", "Max Pool Size", "MaximumPoolSize", "maxpoolsize" },
 			defaultValue: 100u));
 
+		AddOption(DnsCheckInterval = new(
+			keys: new[] { "DNS Check Interval", "DnsCheckInterval" },
+			defaultValue: 0u));
+
 		// Other Options
 		AddOption(AllowLoadLocalInfile = new(
 			keys: new[] { "Allow Load Local Infile", "AllowLoadLocalInfile" },

tests/MySqlConnector.Tests/MySqlConnectionStringBuilderTests.cs

@@ -144,6 +144,7 @@ public void ParseConnectionString()
 				"use xa transactions=false;" +
 				"tls cipher suites=TLS_AES_128_CCM_8_SHA256,TLS_RSA_WITH_RC4_128_MD5;" +
 				"ignore prepare=true;" +
+				"dnscheckinterval=15;" +
 #endif
 				"interactive=true;" +
 				"Keep Alive=90;" +
@@ -207,6 +208,7 @@ public void ParseConnectionString()
 		Assert.False(csb.UseXaTransactions);
 		Assert.Equal("TLS_AES_128_CCM_8_SHA256,TLS_RSA_WITH_RC4_128_MD5", csb.TlsCipherSuites);
 		Assert.True(csb.IgnorePrepare);
+		Assert.Equal(15u, csb.DnsCheckInterval);
 #endif
 		Assert.True(csb.InteractiveSession);
 		Assert.Equal(90u, csb.Keepalive);
@@ -239,7 +241,8 @@ public void ParseConnectionString()
 			"Certificate Store Location=CurrentUser;Certificate Thumbprint=thumbprint123;SSL Cert=client-cert.pem;SSL Key=client-key.pem;" +
 			"SSL CA=ca.pem;TLS Version=\"TLS 1.2, TLS 1.3\";TLS Cipher Suites=TLS_AES_128_CCM_8_SHA256,TLS_RSA_WITH_RC4_128_MD5;" +
 			"Pooling=False;Connection Lifetime=15;Connection Reset=False;Defer Connection Reset=True;Connection Idle Timeout=30;" +
-			"Minimum Pool Size=5;Maximum Pool Size=15;Allow Load Local Infile=True;Allow Public Key Retrieval=True;Allow User Variables=True;" +
+			"Minimum Pool Size=5;Maximum Pool Size=15;DNS Check Interval=15;" +
+			"Allow Load Local Infile=True;Allow Public Key Retrieval=True;Allow User Variables=True;" +
 			"Allow Zero DateTime=True;Application Name=\"My Test Application\";Auto Enlist=False;Cancellation Timeout=-1;Character Set=latin1;" +
 			"Connection Timeout=30;Convert Zero DateTime=True;DateTime Kind=Utc;Default Command Timeout=123;Force Synchronous=True;" +
 			"GUID Format=TimeSwapBinary16;Ignore Command Transaction=True;Ignore Prepare=True;Interactive Session=True;Keep Alive=90;" +
@@ -525,6 +528,7 @@ public void ParseInvalidTlsVersion()
 	[InlineData("Cancellation Timeout", 5)]
 	[InlineData("Connection Idle Timeout", 10u)]
 	[InlineData("DateTime Kind", MySqlDateTimeKind.Utc)]
+	[InlineData("DNS Check Interval", 15u)]
 	[InlineData("Force Synchronous", true)]
 	[InlineData("GUID Format", MySqlGuidFormat.Binary16)]
 	[InlineData("Ignore Command Transaction", true)]