[5.1] Stabilize CI Pipelines (#3599)

* User Story 38467: Backport mac server name fix

- Backported part of #3494 and #3591:
  - Added configurable test jobs timeout, defaulting to 90 minutes.
  - Reduced generated database names to 96 chars to try to fix macOS test failures.

* User Story 38481: Fix unique db object name issues

- Fixed the unique name generators to:
  - Keep max lengths to 30 and 96 characters respectively.
  - Ensure uniqueness at the start of the names.
  - Added link to database identifier syntax.

* User Story 38481: Fix unique db object name issues

- Removed DateOnly tests that aren't supported on 5.1.

* Add CodeQL suppression for DefaultAzureCredential use in Production (#3542)
  - Adjusted CodeQL suppression to meet the strict requirements of where it may appear relative to the flagged code.
  - Adding catch for macOS socket error to log and ignore.

* - Added back some blocks that were removed by cherry-picks.
- Added console diagnostics to see when Enclave tables are dropped.

* - Fixed typo in code that only gets compiled when building on Windows (eek!)

* Tests | Remove hardcoded credentials from ManualTests (#3204)

* Initial removal of CertificateUtility.CreateCertificate

One test implied that DataTestUtility.AKVUrl would point to an RSA key which aligned with the certificate's private key. Switching this to dynamically generate the key in places.

* Hotfix for Azure Key Vault tests

* Removed hardcoded references to Azure Key Vault key

* Removed hardcoded references to CertificateUtilityWin

These were mostly related to generating CSP keys.

* Code review changes

* Reorder properties and constructors
* Move AEConnectionStringProviderWithCspParameters to its own file
* Tweak to the AKV token acquisition

* Code review

Redundant bracket, alphabetised the ManualTesting csproj

* Update src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/SQLSetupStrategy.cs

Let's try @edwardneal's idea

Co-authored-by: Edward Neal <[email protected]>

* Update src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/SQLSetupStrategy.cs

Co-authored-by: Edward Neal <[email protected]>

* Fixes as per @edwardneal's suggestions

* Fix as per @edwardneal's suggestion

* Fix missing `new`

Co-authored-by: Edward Neal <[email protected]>

* Update src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/SQLSetupStrategyAzureKeyVault.cs

Co-authored-by: Edward Neal <[email protected]>

* Update src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/SQLSetupStrategyAzureKeyVault.cs

Co-authored-by: Edward Neal <[email protected]>

* Address comment that we don't need a CspParameters object as part of the test arguments
* Move test arguments into property (the class was only used in a single location)
* Cleanup test code
* Tweak default provider discovery code to handle edge cases a bit better

* Address comment regarding readonly member variables
Apply long line chomping

* Addressing the last of the comments.

---------

Co-authored-by: Edward Neal <[email protected]>

* Update test utilities target frameworks. Fix compilation issues.

* Construct valid X500 distinguished name.

* Print rsa key type for debugging.

* Clean up net version ifdefs to fix certificate exportability.

---------

Co-authored-by: Benjamin Russell <[email protected]>
Co-authored-by: Edward Neal <[email protected]>
Co-authored-by: Malcolm Daigle <[email protected]>

Author: 4 people

eng/pipelines/common/templates/jobs/run-tests-package-reference-job.yml

@@ -17,11 +17,21 @@ parameters:
     type: string
     default: empty
 
+  # The timeout, in minutes, for this job.
+  - name: timeout
+    type: string
+    default: 90
+
 jobs:
 - job: run_tests_package_reference
   displayName: 'Run tests with package reference'
   ${{ if ne(parameters.dependsOn, 'empty')}}:
     dependsOn: '${{parameters.dependsOn }}'
+
+  # Some of our tests take longer than the default 60 minutes to run on some
+  # OSes and configurations.
+  timeoutInMinutes: ${{ parameters.timeout }}
+
   pool:
     type: windows  # read more about custom job pool types at https://aka.ms/obpipelines/yaml/jobs
     isCustom: true

eng/pipelines/dotnet-sqlclient-signing-pipeline.yml

@@ -65,6 +65,12 @@ parameters: # parameters are shown up in ADO UI in a build queue time
   - NonOfficial
   - Official
 
+# The timeout, in minutes, for each test job.
+- name: testsTimeout
+  displayName: 'Tests timeout (in minutes)'
+  type: string
+  default: 90
+
 variables:
   - template: /eng/pipelines/libraries/variables.yml@self
   - name: packageFolderName
@@ -172,6 +178,7 @@ extends:
       - template: eng/pipelines/common/templates/jobs/run-tests-package-reference-job.yml@self
         parameters:
           packageFolderName: $(packageFolderName)
+          timeout: ${{ parameters.testsTimeout }}
           downloadPackageStep:
             download: current
             artifact: $(packageFolderName)

src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/SNI/SNITcpHandle.cs

@@ -851,9 +851,30 @@ public override uint Receive(out SNIPacket packet, int timeoutInMilliseconds)
                 }
                 finally
                 {
-                    // Reset the socket timeout to Timeout.Infinite after the receive operation is done
-                    // to avoid blocking the thread in case of a timeout error.
-                    _socket.ReceiveTimeout = Timeout.Infinite;
+                    const int resetTimeout = Timeout.Infinite;
+
+                    try
+                    {
+                        // Reset the socket timeout to Timeout.Infinite after
+                        // the receive operation is done to avoid blocking the
+                        // thread in case of a timeout error.
+                        _socket.ReceiveTimeout = resetTimeout;
+
+                    }
+                    catch (SocketException ex)
+                    {
+                        // We sometimes see setting the ReceiveTimeout fail
+                        // on macOS. There's isn't much we can do about it
+                        // though, so just log and move on.
+                        SqlClientEventSource.Log.TrySNITraceEvent(
+                            nameof(SNITCPHandle),
+                            EventType.ERR,
+                            "Connection Id {0}, Failed to reset socket " +
+                            "receive timeout to {1}: {2}",
+                            _connectionId,
+                            resetTimeout,
+                            ex.Message);
+                    }
                 }
             }
         }

src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ActiveDirectoryAuthenticationProvider.cs

@@ -587,7 +587,28 @@ private static TokenCredentialData CreateTokenCredentialInstance(TokenCredential
                     defaultAzureCredentialOptions.WorkloadIdentityClientId = tokenCredentialKey._clientId;
                 }
 
-                return new TokenCredentialData(new DefaultAzureCredential(defaultAzureCredentialOptions), GetHash(secret));
+                // SqlClient is a library and provides support to acquire access
+                // token using 'DefaultAzureCredential' on user demand when they
+                // specify 'Authentication = Active Directory Default' in
+                // connection string.
+                //
+                // Default Azure Credential is instantiated by the calling
+                // application when using "Active Directory Default"
+                // authentication code to connect to Azure SQL instance.
+                // SqlClient is a library, doesn't instantiate the credential
+                // without running application instructions.
+                //
+                // Note that CodeQL suppression support can only detect
+                // suppression comments that appear immediately above the
+                // flagged statement, or appended to the end of the statement.
+                // Multi-line justifications are not supported.
+                //
+                // https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/codeql/codeql-semmle#guidance-on-suppressions
+                //
+                // CodeQL [SM05137] See above for justification.
+                DefaultAzureCredential cred = new(defaultAzureCredentialOptions);
+
+                return new TokenCredentialData(cred, GetHash(secret));
             }
 
             TokenCredentialOptions tokenCredentialOptions = new() { AuthorityHost = new Uri(tokenCredentialKey._authority) };

src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/AKVTests.cs

@@ -62,15 +62,15 @@ public void TestEncryptDecryptWithAKV()
         [PlatformSpecific(TestPlatforms.Windows)]
         public void TestRoundTripWithAKVAndCertStoreProvider()
         {
-            using SQLSetupStrategyCertStoreProvider certStoreFixture = new ();
+            SqlColumnEncryptionCertificateStoreProvider certStoreProvider = new SqlColumnEncryptionCertificateStoreProvider();
             byte[] plainTextColumnEncryptionKey = ColumnEncryptionKey.GenerateRandomBytes(ColumnEncryptionKey.KeySizeInBytes);
-            byte[] encryptedColumnEncryptionKeyUsingAKV = _fixture.AkvStoreProvider.EncryptColumnEncryptionKey(DataTestUtility.AKVUrl, @"RSA_OAEP", plainTextColumnEncryptionKey);
-            byte[] columnEncryptionKeyReturnedAKV2Cert = certStoreFixture.CertStoreProvider.DecryptColumnEncryptionKey(certStoreFixture.CspColumnMasterKey.KeyPath, @"RSA_OAEP", encryptedColumnEncryptionKeyUsingAKV);
+            byte[] encryptedColumnEncryptionKeyUsingAKV = _fixture.AkvStoreProvider.EncryptColumnEncryptionKey(_fixture.AkvKeyUrl, @"RSA_OAEP", plainTextColumnEncryptionKey);
+            byte[] columnEncryptionKeyReturnedAKV2Cert = certStoreProvider.DecryptColumnEncryptionKey(_fixture.ColumnMasterKeyPath, @"RSA_OAEP", encryptedColumnEncryptionKeyUsingAKV);
             Assert.True(plainTextColumnEncryptionKey.SequenceEqual(columnEncryptionKeyReturnedAKV2Cert), @"Roundtrip failed");
 
             // Try the opposite.
-            byte[] encryptedColumnEncryptionKeyUsingCert = certStoreFixture.CertStoreProvider.EncryptColumnEncryptionKey(certStoreFixture.CspColumnMasterKey.KeyPath, @"RSA_OAEP", plainTextColumnEncryptionKey);
-            byte[] columnEncryptionKeyReturnedCert2AKV = _fixture.AkvStoreProvider.DecryptColumnEncryptionKey(DataTestUtility.AKVUrl, @"RSA_OAEP", encryptedColumnEncryptionKeyUsingCert);
+            byte[] encryptedColumnEncryptionKeyUsingCert = certStoreProvider.EncryptColumnEncryptionKey(_fixture.ColumnMasterKeyPath, @"RSA_OAEP", plainTextColumnEncryptionKey);
+            byte[] columnEncryptionKeyReturnedCert2AKV = _fixture.AkvStoreProvider.DecryptColumnEncryptionKey(_fixture.AkvKeyUrl, @"RSA_OAEP", encryptedColumnEncryptionKeyUsingCert);
             Assert.True(plainTextColumnEncryptionKey.SequenceEqual(columnEncryptionKeyReturnedCert2AKV), @"Roundtrip failed");
         }