Run MHSM tests weekly, disable attestation in Canary (Azure#35770)

* Only run Managed HSM weekly

* Disable attestation testing in Canary

* Replace matrix configuration

Thanks to @benbp for help figuring this out.

* Use correct key/value separator

Mixing too much JSON and PowerShell. 🤦‍♂️

* Add condition to Microsoft.Web/sites resource

Author: heaths

sdk/keyvault/Azure.Security.KeyVault.Shared/tests/KeyVaultTestEnvironment.cs

@@ -75,7 +75,9 @@ public class KeyVaultTestEnvironment : TestEnvironment
         /// <summary>
         /// Gets the value of the "AZURE_KEYVAULT_ATTESTATION_URL" variable.
         /// </summary>
-        public Uri AttestationUri => new(GetRecordedVariable("AZURE_KEYVAULT_ATTESTATION_URL"), UriKind.Absolute);
+        public Uri AttestationUri => Uri.TryCreate(GetRecordedOptionalVariable("AZURE_KEYVAULT_ATTESTATION_URL"), UriKind.Absolute, out Uri attestationUri)
+            ? attestationUri
+            : throw new IgnoreException("Required variable 'AZURE_KEYVAULT_ATTESTATION_URL' is not defined");
 
         /// <summary>
         /// Throws an <see cref="IgnoreException"/> if <see cref="ManagedHsmUrl"/> is not defined.

sdk/keyvault/platform-matrix.json

@@ -1,6 +1,12 @@
 {
+  "matrix": {
+    "$IMPORT": "eng/pipelines/templates/stages/platform-matrix.json",
+    "ArmTemplateParameters": "@{ enableAttestation = $true }"
+  },
   "displayNames": {
-      "@{ enableHsm = $true }": "HSM"
+    "@{ enableAttestation = $true }": "",
+    "@{ enableAttestation = $false }": "NoAttestation",
+    "@{ enableHsm = $true }": "HSM"
   },
   "include": [
     {

sdk/keyvault/test-resources.json

@@ -69,6 +69,13 @@
                 "description": "The location of the Managed HSM. By default, this is 'northcentralus'."
             }
         },
+        "enableAttestation": {
+            "type": "bool",
+            "defaultValue": true,
+            "metadata": {
+                "description": "Whether to enable deployment of attestation resources. The default is true."
+            }
+        },
         "enableHsm": {
             "type": "bool",
             "defaultValue": false,
@@ -223,6 +230,7 @@
             "type": "Microsoft.Web/serverfarms",
             "apiVersion": "2020-12-01",
             "name": "[variables('attestationFarm')]",
+            "condition": "[parameters('enableAttestation')]",
             "location": "[parameters('location')]",
             "kind": "linux",
             "sku": {
@@ -237,6 +245,7 @@
             "type": "Microsoft.Web/sites",
             "apiVersion": "2020-12-01",
             "name": "[variables('attestationSite')]",
+            "condition": "[parameters('enableAttestation')]",
             "dependsOn": [
                 "[resourceId('Microsoft.Web/serverfarms', variables('attestationFarm'))]"
             ],
@@ -290,6 +299,7 @@
         },
         "AZURE_KEYVAULT_ATTESTATION_URL": {
             "type": "string",
+            "condition": "[parameters('enableAttestation')]",
             "value": "[format('https://{0}/', reference(variables('attestationSite')).defaultHostName)]"
         }
     }

sdk/keyvault/tests.yml

@@ -10,23 +10,29 @@ extends:
     CloudConfig:
       Public:
         SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
+        ${{ if not(contains(variables['Build.DefinitionName'], 'tests-weekly')) }}:
+          MatrixFilters:
+            - ArmTemplateParameters=^(?!.*enableHsm.*true)
       Canary:
         SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
         Location: 'eastus2euap'
         # Managed HSM test resources are expensive and provisioning has not been reliable.
         # Given test coverage of non-canary regions we probably don't need to test in canary.
         MatrixFilters:
          - ArmTemplateParameters=^(?!.*enableHsm.*true)
+        # Some resource providers required for attestation are not supported in canary.
+        MatrixReplace:
+         - 'ArmTemplateParameters=(.*)enableAttestation.*?\$true(.*)/$1enableAttestation \= $false$2'
       UsGov:
         SubscriptionConfiguration: $(sub-config-gov-test-resources)
         MatrixFilters:
          - ArmTemplateParameters=^(?!.*enableHsm.*true)
       China:
         SubscriptionConfiguration: $(sub-config-cn-test-resources)
         MatrixFilters:
-         - ArmTemplateParameters=^(?!.*enableHsm.*true)
-    AdditionalMatrixConfigs:
-      - Name: keyvault_test_matrix_addons
+          - ArmTemplateParameters=^(?!.*enableHsm.*true)
+    MatrixConfigs:
+      - Name: keyvault_test_matrix
         Path: sdk/keyvault/platform-matrix.json
         Selection: sparse
         GenerateVMJobs: true