diff --git a/src/Shared/CertificateGeneration/CertificateManager.cs b/src/Shared/CertificateGeneration/CertificateManager.cs
index aba1240c7b12..bd7104542fab 100644
--- a/src/Shared/CertificateGeneration/CertificateManager.cs
+++ b/src/Shared/CertificateGeneration/CertificateManager.cs
@@ -236,18 +236,22 @@ public EnsureCertificateResult EnsureAspNetCoreHttpsDevelopmentCertificate(
 
         var currentUserCertificates = ListCertificates(StoreName.My, StoreLocation.CurrentUser, isValid: true, requireExportable: true);
         var localMachineCertificates = ListCertificates(StoreName.My, StoreLocation.LocalMachine, isValid: true, requireExportable: true);
-        var certificates = currentUserCertificates.Concat(localMachineCertificates);
+        var allCertificates = currentUserCertificates.Concat(localMachineCertificates).ToList();
 
-        var filteredCertificates = certificates.Where(c => c.Subject == Subject);
+        var filteredCertificates = allCertificates.Where(c => c.Subject == Subject).ToList();
 
         if (Log.IsEnabled())
         {
-            var excludedCertificates = certificates.Except(filteredCertificates);
+            var excludedCertificates = allCertificates.Except(filteredCertificates);
             Log.FilteredCertificates(ToCertificateDescription(filteredCertificates));
             Log.ExcludedCertificates(ToCertificateDescription(excludedCertificates));
         }
 
-        certificates = filteredCertificates;
+        // Dispose certificates we're not going to use
+        var certificatesToDispose = allCertificates.Except(filteredCertificates);
+        DisposeCertificates(certificatesToDispose);
+
+        var certificates = filteredCertificates;
 
         X509Certificate2? certificate = null;
         var isNewCertificate = false;
@@ -319,6 +323,7 @@ public EnsureCertificateResult EnsureAspNetCoreHttpsDevelopmentCertificate(
                     Log.CreateDevelopmentCertificateError(e.ToString());
                 }
                 result = EnsureCertificateResult.ErrorCreatingTheCertificate;
+                DisposeCertificates(certificates);
                 return result;
             }
             Log.CreateDevelopmentCertificateEnd();
@@ -330,6 +335,11 @@ public EnsureCertificateResult EnsureAspNetCoreHttpsDevelopmentCertificate(
             catch (Exception e)
             {
                 Log.SaveCertificateInStoreError(e.ToString());
+                if (isNewCertificate)
+                {
+                    certificate?.Dispose();
+                }
+                DisposeCertificates(certificates);
                 result = EnsureCertificateResult.ErrorSavingTheCertificateIntoTheCurrentUserPersonalStore;
                 return result;
             }
@@ -387,6 +397,11 @@ public EnsureCertificateResult EnsureAspNetCoreHttpsDevelopmentCertificate(
                     result :
                     EnsureCertificateResult.ErrorExportingTheCertificate;
 
+                if (isNewCertificate)
+                {
+                    certificate?.Dispose();
+                }
+                DisposeCertificates(certificates);
                 return result;
             }
         }
@@ -403,21 +418,41 @@ public EnsureCertificateResult EnsureAspNetCoreHttpsDevelopmentCertificate(
                         break;
                     case TrustLevel.Partial:
                         result = EnsureCertificateResult.PartiallyFailedToTrustTheCertificate;
+                        if (isNewCertificate)
+                        {
+                            certificate?.Dispose();
+                        }
+                        DisposeCertificates(certificates);
                         return result;
                     case TrustLevel.None:
                     default: // Treat unknown status (should be impossible) as failure
                         result = EnsureCertificateResult.FailedToTrustTheCertificate;
+                        if (isNewCertificate)
+                        {
+                            certificate?.Dispose();
+                        }
+                        DisposeCertificates(certificates);
                         return result;
                 }
             }
             catch (UserCancelledTrustException)
             {
                 result = EnsureCertificateResult.UserCancelledTrustStep;
+                if (isNewCertificate)
+                {
+                    certificate?.Dispose();
+                }
+                DisposeCertificates(certificates);
                 return result;
             }
             catch
             {
                 result = EnsureCertificateResult.FailedToTrustTheCertificate;
+                if (isNewCertificate)
+                {
+                    certificate?.Dispose();
+                }
+                DisposeCertificates(certificates);
                 return result;
             }
 
@@ -454,51 +489,58 @@ internal ImportCertificateResult ImportCertificate(string certificatePath, strin
             return ImportCertificateResult.ExistingCertificatesPresent;
         }
 
-        X509Certificate2 certificate;
+        X509Certificate2? certificate = null;
         try
         {
-            Log.LoadCertificateStart(certificatePath);
-            certificate = new X509Certificate2(certificatePath, password, X509KeyStorageFlags.Exportable | X509KeyStorageFlags.EphemeralKeySet);
-            if (Log.IsEnabled())
+            try
             {
-                Log.LoadCertificateEnd(GetDescription(certificate));
+                Log.LoadCertificateStart(certificatePath);
+                certificate = new X509Certificate2(certificatePath, password, X509KeyStorageFlags.Exportable | X509KeyStorageFlags.EphemeralKeySet);
+                if (Log.IsEnabled())
+                {
+                    Log.LoadCertificateEnd(GetDescription(certificate));
+                }
             }
-        }
-        catch (Exception e)
-        {
-            if (Log.IsEnabled())
+            catch (Exception e)
             {
-                Log.LoadCertificateError(e.ToString());
+                if (Log.IsEnabled())
+                {
+                    Log.LoadCertificateError(e.ToString());
+                }
+                return ImportCertificateResult.InvalidCertificate;
             }
-            return ImportCertificateResult.InvalidCertificate;
-        }
 
-        // Note that we're checking Subject, rather than LocalhostHttpsDistinguishedName,
-        // because the tests use a different subject.
-        if (!string.Equals(certificate.Subject, Subject, StringComparison.Ordinal) || // Kestrel requires this
-            !IsHttpsDevelopmentCertificate(certificate))
-        {
-            if (Log.IsEnabled())
+            // Note that we're checking Subject, rather than LocalhostHttpsDistinguishedName,
+            // because the tests use a different subject.
+            if (!string.Equals(certificate.Subject, Subject, StringComparison.Ordinal) || // Kestrel requires this
+                !IsHttpsDevelopmentCertificate(certificate))
             {
-                Log.NoHttpsDevelopmentCertificate(GetDescription(certificate));
+                if (Log.IsEnabled())
+                {
+                    Log.NoHttpsDevelopmentCertificate(GetDescription(certificate));
+                }
+                return ImportCertificateResult.NoDevelopmentHttpsCertificate;
             }
-            return ImportCertificateResult.NoDevelopmentHttpsCertificate;
-        }
 
-        try
-        {
-            SaveCertificate(certificate);
-        }
-        catch (Exception e)
-        {
-            if (Log.IsEnabled())
+            try
             {
-                Log.SaveCertificateInStoreError(e.ToString());
+                certificate = SaveCertificate(certificate);
+            }
+            catch (Exception e)
+            {
+                if (Log.IsEnabled())
+                {
+                    Log.SaveCertificateInStoreError(e.ToString());
+                }
+                return ImportCertificateResult.ErrorSavingTheCertificateIntoTheCurrentUserPersonalStore;
             }
-            return ImportCertificateResult.ErrorSavingTheCertificateIntoTheCurrentUserPersonalStore;
-        }
 
-        return ImportCertificateResult.Succeeded;
+            return ImportCertificateResult.Succeeded;
+        }
+        finally
+        {
+            certificate?.Dispose();
+        }
     }
 
     public void CleanupHttpsCertificates()