Merge dns-check-interval into master.

bgrainger · bgrainger · commit 660bf82833d8 · 2022-09-18T16:24:26.000-07:00
diff --git a/docs/content/connection-options.md b/docs/content/connection-options.md
@@ -232,6 +232,15 @@ Connection pooling is enabled by default. These options are used to configure it
     <td>100</td>
     <td>The maximum number of connections allowed in the pool.</td>
   </tr>
+  <tr id="DnsCheckInterval">
+    <td>DNS Check Interval, DnsCheckInterval</td>
+    <td>0</td>
+    <td>The number of seconds between checks for DNS changes, or 0 to disable periodic checks.
+    If the periodic check determines that one of the <code>Server</code> hostnames resolves to a different IP address, the pool will be cleared.
+    This is useful in HA scenarios where failover is accomplished by changing the IP address to which a hostname resolves.
+    Existing connections in the pool may have valid TCP connections to a server that is no longer responding or has been marked readonly;
+    clearing the pool (when DNS changes) forces all these existing connections to be reestablished.</td>
+  </tr>
 </table>
 
 ## Other Options
diff --git a/src/MySqlConnector/Core/ConnectionPool.cs b/src/MySqlConnector/Core/ConnectionPool.cs
@@ -1,12 +1,14 @@
 using System.Collections.Concurrent;
+using System.Diagnostics;
+using System.Net;
 using System.Security.Authentication;
 using MySqlConnector.Logging;
 using MySqlConnector.Protocol.Serialization;
 using MySqlConnector.Utilities;
 
 namespace MySqlConnector.Core;
 
-internal sealed class ConnectionPool
+internal sealed class ConnectionPool : IDisposable
 {
 	public int Id { get; }
 
@@ -219,6 +221,30 @@ public async Task ReapAsync(IOBehavior ioBehavior, CancellationToken cancellatio
 		return procedureCache;
 	}
 
+	public void Dispose()
+	{
+		Log.Debug("Pool{0} disposing connection pool", m_logArguments);
+#if NET6_0_OR_GREATER
+		m_dnsCheckTimer?.Dispose();
+		m_dnsCheckTimer = null;
+		m_reaperTimer?.Dispose();
+		m_reaperTimer = null;
+#else
+		if (m_dnsCheckTimer is not null)
+		{
+			using var dnsCheckWaitHandle = new ManualResetEvent(false);
+			m_dnsCheckTimer.Dispose(dnsCheckWaitHandle);
+			dnsCheckWaitHandle.WaitOne();
+		}
+		if (m_reaperTimer is not null)
+		{
+			using var reaperWaitHandle = new ManualResetEvent(false);
+			m_reaperTimer.Dispose(reaperWaitHandle);
+			reaperWaitHandle.WaitOne();
+		}
+#endif
+	}
+
 	/// <summary>
 	/// Examines all the <see cref="ServerSession"/> objects in <see cref="m_leasedSessions"/> to determine if any
 	/// have an owning <see cref="MySqlConnection"/> that has been garbage-collected. If so, assumes that the connection
@@ -438,6 +464,7 @@ private async ValueTask<ServerSession> ConnectSessionAsync(MySqlConnection conne
 		var connectionSettings = new ConnectionSettings(connectionStringBuilder);
 		var pool = new ConnectionPool(connectionSettings);
 		pool.StartReaperTask();
+		pool.StartDnsCheckTimer();
 		return pool;
 	}
 
@@ -485,6 +512,7 @@ private async ValueTask<ServerSession> ConnectSessionAsync(MySqlConnection conne
 		{
 			s_mruCache = new(connectionString, pool);
 			pool.StartReaperTask();
+			pool.StartDnsCheckTimer();
 
 			// if we won the race to create the new pool, also store it under the original connection string
 			if (connectionString != normalizedConnectionString)
@@ -546,27 +574,136 @@ private ConnectionPool(ConnectionSettings cs)
 
 	private void StartReaperTask()
 	{
-		if (ConnectionSettings.ConnectionIdleTimeout > 0)
+		if (ConnectionSettings.ConnectionIdleTimeout <= 0)
+			return;
+
+		var reaperInterval = TimeSpan.FromSeconds(Math.Max(1, Math.Min(60, ConnectionSettings.ConnectionIdleTimeout / 2)));
+
+#if NET6_0_OR_GREATER
+		m_reaperTimer = new PeriodicTimer(reaperInterval);
+		_ = RunTimer();
+
+		async Task RunTimer()
+		{
+			while (await m_reaperTimer.WaitForNextTickAsync().ConfigureAwait(false))
+			{
+				try
+				{
+					using var source = new CancellationTokenSource(reaperInterval);
+					await ReapAsync(IOBehavior.Asynchronous, source.Token).ConfigureAwait(false);
+				}
+				catch
+				{
+					// do nothing; we'll try to reap again
+				}
+			}
+		}
+#else
+		m_reaperTimer = new Timer(t =>
+		{
+			var stopwatch = Stopwatch.StartNew();
+			try
+			{
+				using var source = new CancellationTokenSource(reaperInterval);
+				ReapAsync(IOBehavior.Synchronous, source.Token).GetAwaiter().GetResult();
+			}
+			catch
+			{
+				// do nothing; we'll try to reap again
+			}
+
+			// restart the timer, accounting for the time spent reaping
+			var delay = reaperInterval - stopwatch.Elapsed;
+			((Timer) t!).Change(delay < TimeSpan.Zero ? TimeSpan.Zero : delay, TimeSpan.FromMilliseconds(-1));
+		});
+		m_reaperTimer.Change(reaperInterval, TimeSpan.FromMilliseconds(-1));
+#endif
+	}
+
+	private void StartDnsCheckTimer()
+	{
+		if (ConnectionSettings.ConnectionProtocol != MySqlConnectionProtocol.Tcp || ConnectionSettings.DnsCheckInterval <= 0)
+			return;
+
+		var hostNames = ConnectionSettings.HostNames!;
+		var hostAddresses = new IPAddress[hostNames.Count][];
+
+#if NET6_0_OR_GREATER
+		m_dnsCheckTimer = new PeriodicTimer(TimeSpan.FromSeconds(ConnectionSettings.DnsCheckInterval));
+		_ = RunTimer();
+
+		async Task RunTimer()
 		{
-			var reaperInterval = TimeSpan.FromSeconds(Math.Max(1, Math.Min(60, ConnectionSettings.ConnectionIdleTimeout / 2)));
-			m_reaperTask = Task.Run(async () =>
+			while (await m_dnsCheckTimer.WaitForNextTickAsync().ConfigureAwait(false))
 			{
-				while (true)
+				Log.Trace("Pool{0} checking for DNS changes", m_logArguments);
+				var hostNamesChanged = false;
+				for (var hostNameIndex = 0; hostNameIndex < hostNames.Count; hostNameIndex++)
 				{
-					var task = Task.Delay(reaperInterval);
 					try
 					{
-						using var source = new CancellationTokenSource(reaperInterval);
-						await ReapAsync(IOBehavior.Asynchronous, source.Token).ConfigureAwait(false);
+						var ipAddresses = await Dns.GetHostAddressesAsync(hostNames[hostNameIndex]).ConfigureAwait(false);
+						if (hostAddresses[hostNameIndex] is null)
+						{
+							hostAddresses[hostNameIndex] = ipAddresses;
+						}
+						else if (hostAddresses[hostNameIndex].Except(ipAddresses).Any())
+						{
+							Log.Debug("Pool{0} detected DNS change for HostName '{1}': {2} to {3}", m_logArguments[0], hostNames[hostNameIndex], string.Join<IPAddress>(',', hostAddresses[hostNameIndex]), string.Join<IPAddress>(',', ipAddresses));
+							hostAddresses[hostNameIndex] = ipAddresses;
+							hostNamesChanged = true;
+						}
 					}
-					catch
+					catch (Exception ex)
 					{
-						// do nothing; we'll try to reap again
+						// do nothing; we'll try again later
+						Log.Debug("Pool{0} DNS check failed; ignoring HostName '{1}': {2}", m_logArguments[0], hostNames[hostNameIndex], ex.Message);
 					}
-					await task.ConfigureAwait(false);
 				}
-			});
+				if (hostNamesChanged)
+				{
+					Log.Info("Pool{0} clearing pool due to DNS changes", m_logArguments);
+					await ClearAsync(IOBehavior.Asynchronous, CancellationToken.None).ConfigureAwait(false);
+				}
+			}
 		}
+#else
+		var interval = Math.Min(int.MaxValue / 1000, ConnectionSettings.DnsCheckInterval) * 1000;
+		m_dnsCheckTimer = new Timer(t =>
+		{
+			Log.Trace("Pool{0} checking for DNS changes", m_logArguments);
+			var hostNamesChanged = false;
+			for (var hostNameIndex = 0; hostNameIndex < hostNames.Count; hostNameIndex++)
+			{
+				try
+				{
+					var ipAddresses = Dns.GetHostAddresses(hostNames[hostNameIndex]);
+					if (hostAddresses[hostNameIndex] is null)
+					{
+						hostAddresses[hostNameIndex] = ipAddresses;
+					}
+					else if (hostAddresses[hostNameIndex].Except(ipAddresses).Any())
+					{
+						Log.Debug("Pool{0} detected DNS change for HostName '{1}': {2} to {3}", m_logArguments[0], hostNames[hostNameIndex], string.Join<IPAddress>(",", hostAddresses[hostNameIndex]), string.Join<IPAddress>(",", ipAddresses));
+						hostAddresses[hostNameIndex] = ipAddresses;
+						hostNamesChanged = true;
+					}
+				}
+				catch (Exception ex)
+				{
+					// do nothing; we'll try again later
+					Log.Debug("Pool{0} DNS check failed; ignoring HostName '{1}': {2}", m_logArguments[0], hostNames[hostNameIndex], ex.Message);
+				}
+			}
+			if (hostNamesChanged)
+			{
+				Log.Info("Pool{0} clearing pool due to DNS changes", m_logArguments);
+				ClearAsync(IOBehavior.Synchronous, CancellationToken.None).GetAwaiter().GetResult();
+			}
+			((Timer) t!).Change(interval, -1);
+		});
+		m_dnsCheckTimer.Change(interval, -1);
+#endif
 	}
 
 	private void AdjustHostConnectionCount(ServerSession session, int delta)
@@ -626,8 +763,14 @@ private static void OnAppDomainShutDown(object? sender, EventArgs e) =>
 	private readonly Dictionary<string, int>? m_hostSessions;
 	private readonly object[] m_logArguments;
 	private int m_generation;
-	private Task? m_reaperTask;
 	private uint m_lastRecoveryTime;
 	private int m_lastSessionId;
 	private Dictionary<string, CachedProcedure?>? m_procedureCache;
+#if NET6_0_OR_GREATER
+	private PeriodicTimer? m_dnsCheckTimer;
+	private PeriodicTimer? m_reaperTimer;
+#else
+	private Timer? m_dnsCheckTimer;
+	private Timer? m_reaperTimer;
+#endif
 }
diff --git a/src/MySqlConnector/Core/ConnectionSettings.cs b/src/MySqlConnector/Core/ConnectionSettings.cs
@@ -119,6 +119,7 @@ public ConnectionSettings(MySqlConnectionStringBuilder csb)
 			throw new MySqlException("MaximumPoolSize must be greater than or equal to MinimumPoolSize");
 		MinimumPoolSize = ToSigned(csb.MinimumPoolSize);
 		MaximumPoolSize = ToSigned(csb.MaximumPoolSize);
+		DnsCheckInterval = ToSigned(csb.DnsCheckInterval);
 
 		// Other Options
 		AllowLoadLocalInfile = csb.AllowLoadLocalInfile;
@@ -213,6 +214,7 @@ private static MySqlGuidFormat GetEffectiveGuidFormat(MySqlGuidFormat guidFormat
 	public int ConnectionIdleTimeout { get; }
 	public int MinimumPoolSize { get; }
 	public int MaximumPoolSize { get; }
+	public int DnsCheckInterval { get; }
 
 	// Other Options
 	public bool AllowLoadLocalInfile { get; }
@@ -299,6 +301,7 @@ private ConnectionSettings(ConnectionSettings other, string host, int port, stri
 		ConnectionIdleTimeout = other.ConnectionIdleTimeout;
 		MinimumPoolSize = other.MinimumPoolSize;
 		MaximumPoolSize = other.MaximumPoolSize;
+		DnsCheckInterval = other.DnsCheckInterval;
 
 		AllowLoadLocalInfile = other.AllowLoadLocalInfile;
 		AllowPublicKeyRetrieval = other.AllowPublicKeyRetrieval;
diff --git a/src/MySqlConnector/MySqlConnectionStringBuilder.cs b/src/MySqlConnector/MySqlConnectionStringBuilder.cs
@@ -404,6 +404,19 @@ public uint MaximumPoolSize
 		set => MySqlConnectionStringOption.MaximumPoolSize.SetValue(this, value);
 	}
 
+	/// <summary>
+	/// The number of seconds between checks for DNS changes, or 0 to disable periodic checks.
+	/// </summary>
+	[Category("Pooling")]
+	[DefaultValue(0u)]
+	[Description("The number of seconds between checks for DNS changes.")]
+	[DisplayName("DNS Check Interval")]
+	public uint DnsCheckInterval
+	{
+		get => MySqlConnectionStringOption.DnsCheckInterval.GetValue(this);
+		set => MySqlConnectionStringOption.DnsCheckInterval.SetValue(this, value);
+	}
+
 	// Other Options
 
 	/// <summary>
@@ -915,6 +928,7 @@ internal abstract partial class MySqlConnectionStringOption
 	public static readonly MySqlConnectionStringValueOption<uint> ConnectionIdleTimeout;
 	public static readonly MySqlConnectionStringValueOption<uint> MinimumPoolSize;
 	public static readonly MySqlConnectionStringValueOption<uint> MaximumPoolSize;
+	public static readonly MySqlConnectionStringValueOption<uint> DnsCheckInterval;
 
 	// Other Options
 	public static readonly MySqlConnectionStringValueOption<bool> AllowLoadLocalInfile;
@@ -1118,6 +1132,10 @@ static MySqlConnectionStringOption()
 			keys: new[] { "Maximum Pool Size", "Max Pool Size", "MaximumPoolSize", "maxpoolsize" },
 			defaultValue: 100u));
 
+		AddOption(DnsCheckInterval = new(
+			keys: new[] { "DNS Check Interval", "DnsCheckInterval" },
+			defaultValue: 0u));
+
 		// Other Options
 		AddOption(AllowLoadLocalInfile = new(
 			keys: new[] { "Allow Load Local Infile", "AllowLoadLocalInfile" },
diff --git a/src/MySqlConnector/MySqlDataSource.cs b/src/MySqlConnector/MySqlDataSource.cs
@@ -91,7 +91,10 @@ protected override void Dispose(bool disposing)
 	private async ValueTask DisposeAsync(IOBehavior ioBehavior)
 	{
 		if (Pool is not null)
+		{
 			await Pool.ClearAsync(ioBehavior, default).ConfigureAwait(false);
+			Pool.Dispose();
+		}
 		m_isDisposed = true;
 	}
 
diff --git a/tests/MySqlConnector.Tests/MySqlConnectionStringBuilderTests.cs b/tests/MySqlConnector.Tests/MySqlConnectionStringBuilderTests.cs
@@ -144,6 +144,7 @@ public void ParseConnectionString()
 				"use xa transactions=false;" +
 				"tls cipher suites=TLS_AES_128_CCM_8_SHA256,TLS_RSA_WITH_RC4_128_MD5;" +
 				"ignore prepare=true;" +
+				"dnscheckinterval=15;" +
 #endif
 				"interactive=true;" +
 				"Keep Alive=90;" +
@@ -207,6 +208,7 @@ public void ParseConnectionString()
 		Assert.False(csb.UseXaTransactions);
 		Assert.Equal("TLS_AES_128_CCM_8_SHA256,TLS_RSA_WITH_RC4_128_MD5", csb.TlsCipherSuites);
 		Assert.True(csb.IgnorePrepare);
+		Assert.Equal(15u, csb.DnsCheckInterval);
 #endif
 		Assert.True(csb.InteractiveSession);
 		Assert.Equal(90u, csb.Keepalive);
@@ -239,7 +241,8 @@ public void ParseConnectionString()
 			"Certificate Store Location=CurrentUser;Certificate Thumbprint=thumbprint123;SSL Cert=client-cert.pem;SSL Key=client-key.pem;" +
 			"SSL CA=ca.pem;TLS Version=\"TLS 1.2, TLS 1.3\";TLS Cipher Suites=TLS_AES_128_CCM_8_SHA256,TLS_RSA_WITH_RC4_128_MD5;" +
 			"Pooling=False;Connection Lifetime=15;Connection Reset=False;Defer Connection Reset=True;Connection Idle Timeout=30;" +
-			"Minimum Pool Size=5;Maximum Pool Size=15;Allow Load Local Infile=True;Allow Public Key Retrieval=True;Allow User Variables=True;" +
+			"Minimum Pool Size=5;Maximum Pool Size=15;DNS Check Interval=15;" +
+			"Allow Load Local Infile=True;Allow Public Key Retrieval=True;Allow User Variables=True;" +
 			"Allow Zero DateTime=True;Application Name=\"My Test Application\";Auto Enlist=False;Cancellation Timeout=-1;Character Set=latin1;" +
 			"Connection Timeout=30;Convert Zero DateTime=True;DateTime Kind=Utc;Default Command Timeout=123;Force Synchronous=True;" +
 			"GUID Format=TimeSwapBinary16;Ignore Command Transaction=True;Ignore Prepare=True;Interactive Session=True;Keep Alive=90;" +
@@ -525,6 +528,7 @@ public void ParseInvalidTlsVersion()
 	[InlineData("Cancellation Timeout", 5)]
 	[InlineData("Connection Idle Timeout", 10u)]
 	[InlineData("DateTime Kind", MySqlDateTimeKind.Utc)]
+	[InlineData("DNS Check Interval", 15u)]
 	[InlineData("Force Synchronous", true)]
 	[InlineData("GUID Format", MySqlGuidFormat.Binary16)]
 	[InlineData("Ignore Command Transaction", true)]

Original file line number	Diff line number	Diff line change
`@@ -91,7 +91,10 @@ protected override void Dispose(bool disposing)`
`91`	`91`	`private async ValueTask DisposeAsync(IOBehavior ioBehavior)`
`92`	`92`	`{`
`93`	`93`	`if (Pool is not null)`
	`94`	`+ {`
`94`	`95`	`await Pool.ClearAsync(ioBehavior, default).ConfigureAwait(false);`
	`96`	`+ Pool.Dispose();`
	`97`	`+ }`
`95`	`98`	`m_isDisposed = true;`
`96`	`99`	`}`
`97`	`100`