Support Custom Ambari DB as metastore

Zhenyu Zhou · Zhenyu Zhou · commit d20e5ac605e1 · 2020-11-05T14:29:48.000+08:00
diff --git a/src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightClusterTests.cs b/src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightClusterTests.cs
@@ -84,5 +84,12 @@ public void TestCreateClusterWithRelayOutoundAndPrivateLink()
         {
             TestController.NewInstance.RunPowerShellTest(_logger, "Test-CreateClusterWithRelayOutoundAndPrivateLink");
         }
+
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void TestCreateClusterWithCustomAmbariDatabase()
+        {
+            TestController.NewInstance.RunPowerShellTest(_logger, "Test-CreateClusterWithCustomAmbariDatabase");
+        }
     }
 }
diff --git a/src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightClusterTests.ps1 b/src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightClusterTests.ps1
@@ -313,3 +313,44 @@ function Test-CreateClusterWithRelayOutoundAndPrivateLink{
 		Remove-AzResourceGroup -ResourceGroupName $cluster.ResourceGroup
 	}
 }
+
+<#
+.SYNOPSIS
+Test Create Azure HDInsight Cluster with custom ambari database
+#>
+
+function Test-CreateClusterWithCustomAmbariDatabase{
+
+	# Create some resources that will be used throughout test
+	try
+	{
+		# prepare parameter for creating parameter
+		$params= Prepare-ClusterCreateParameter -location "South Central US"
+
+		# prepare custom ambari database
+		$databaseUserName="databaseuser"
+		$databasePassword="xxxxxxx"
+		$databasePassword=ConvertTo-SecureString $databasePassword -AsPlainText -Force
+		$sqlserverCredential=New-Object System.Management.Automation.PSCredential($databaseUserName, $databasePassword)
+		$sqlserver="yoursqlserver.database.windows.net"
+		$database="yourdatabase"
+		$config=New-AzHDInsightClusterConfig
+
+		# create cluster
+		$cluster = New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
+		-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
+		-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
+		-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
+		-MinSupportedTlsVersion $params.minSupportedTlsVersion `
+		-AmbariDatabase $config.AmbariDatabase
+
+		Assert-NotNull $cluster
+
+	}
+	finally
+	{
+		# Delete cluster and resource group
+		Remove-AzHDInsightCluster -ClusterName $cluster.Name
+		Remove-AzResourceGroup -ResourceGroupName $cluster.ResourceGroup
+	}
+}
diff --git a/src/HDInsight/HDInsight.Test/SessionRecords/Commands.HDInsight.Test.ScenarioTests.HDInsightClusterTests/TestCreateClusterWithCustomAmbariDatabase.json b/src/HDInsight/HDInsight.Test/SessionRecords/Commands.HDInsight.Test.ScenarioTests.HDInsightClusterTests/TestCreateClusterWithCustomAmbariDatabase.json
diff --git a/src/HDInsight/HDInsight/ChangeLog.md b/src/HDInsight/HDInsight/ChangeLog.md
@@ -20,6 +20,8 @@
 ## Upcoming Release
 
 * Add parameters `ResourceProviderConnection` and `PrivateLink` to cmdlet `New-AzHDInsightCluster` to support relay outbound and private link feature
+* Add parameter `AmbariDatabase` to cmdlet `New-AzHDInsightCluster` to support custom Ambari database feature
+* Add accept value "AmbariDatabase" to the parameter `MetastoreType` of the cmdlet `Add-AzHDInsightMetastore`
 
 ## Version 4.0.0
  * For New-AzHDInsightCluster cmdlet:
diff --git a/src/HDInsight/HDInsight/ClusterCreateHelper.cs b/src/HDInsight/HDInsight/ClusterCreateHelper.cs
@@ -11,7 +11,6 @@
 // limitations under the License.
 // ----------------------------------------------------------------------------------
 
-using Microsoft.Azure.Commands.HDInsight.Models.Management;
 using Microsoft.Azure.Management.HDInsight.Models;
 using Microsoft.WindowsAzure.Commands.Common;
 using System;
@@ -25,7 +24,7 @@ public static class ClusterCreateHelper
     {
         public static void AddClusterCredentialToGatewayConfig(PSCredential httpCredential, IDictionary<string, Dictionary<string, string>> configurations)
         {
-            Dictionary<string, string> gatewayConfig = GetExistingConfigurationsForType(configurations, ConfigurationKey.Gateway);
+            Dictionary<string, string> gatewayConfig = GetExistingConfigurationsForType(configurations, Constants.ConfigurationKey.Gateway);
             if (!string.IsNullOrEmpty(httpCredential?.UserName))
             {
                 gatewayConfig[Constants.GatewayConfigurations.CredentialIsEnabledKey] = "true";
@@ -37,7 +36,7 @@ public static void AddClusterCredentialToGatewayConfig(PSCredential httpCredenti
                 gatewayConfig[Constants.GatewayConfigurations.CredentialIsEnabledKey] = "false";
             }
 
-            configurations[ConfigurationKey.Gateway] = gatewayConfig;
+            configurations[Constants.ConfigurationKey.Gateway] = gatewayConfig;
         }
 
         public static void AddAzureDataLakeStorageGen1ToCoreConfig(string storageResourceId, string storageRootPath, string defaultAzureDataLakeStoreFileSystemEndpointSuffix, IDictionary<string, Dictionary<string, string>> configurations)
@@ -56,28 +55,28 @@ public static void AddAzureDataLakeStorageGen1ToCoreConfig(string storageResourc
             }
 
             // Get existing core configs.
-            Dictionary<string, string> coreConfig = GetExistingConfigurationsForType(configurations, ConfigurationKey.CoreSite);
+            Dictionary<string, string> coreConfig = GetExistingConfigurationsForType(configurations, Constants.ConfigurationKey.CoreSite);
 
             // Add configurations for default ADL storage.
             coreConfig[Constants.StorageConfigurations.DefaultFsKey] = Constants.StorageConfigurations.DefaultFsAdlValue;
             coreConfig[Constants.StorageConfigurations.AdlHostNameKey] = storageAccountName;
             coreConfig[Constants.StorageConfigurations.AdlMountPointKey] = storageRootPath;
 
-            configurations[ConfigurationKey.CoreSite] = coreConfig;
+            configurations[Constants.ConfigurationKey.CoreSite] = coreConfig;
         }
 
         public static void AddAdditionalStorageAccountsToCoreConfig(Dictionary<string, string> additionalStorageAccounts, IDictionary<string, Dictionary<string, string>> configurations)
         {
             // Get existing core configs.
-            Dictionary<string, string> coreConfig = GetExistingConfigurationsForType(configurations, ConfigurationKey.CoreSite);
+            Dictionary<string, string> coreConfig = GetExistingConfigurationsForType(configurations, Constants.ConfigurationKey.CoreSite);
 
             foreach (KeyValuePair<string, string> storageAccount in additionalStorageAccounts)
             {
                 string configKey = string.Format(Constants.StorageConfigurations.WasbStorageAccountKeyFormat, storageAccount.Key);
                 coreConfig[configKey] = storageAccount.Value;
             }
 
-            configurations[ConfigurationKey.CoreSite] = coreConfig;
+            configurations[Constants.ConfigurationKey.CoreSite] = coreConfig;
         }
 
         public static void AddDataLakeStorageGen1IdentityToIdentityConfig(Guid applicationId, Guid aadTenantId, byte[] certificateFileBytes, string certificatePassword,
@@ -95,7 +94,7 @@ public static void AddDataLakeStorageGen1IdentityToIdentityConfig(Guid applicati
                 {Constants.DataLakeConfigurations.ResourceUriKey, dataLakeEndpointResourceId}
             };
 
-            configurations[ConfigurationKey.ClusterIdentity] = datalakeConfig;
+            configurations[Constants.ConfigurationKey.ClusterIdentity] = datalakeConfig;
         }
 
         public static StorageAccount CreateAzureStorageAccount(string clusterName, string storageResourceId, string storageAccountkey, string storageContainer, string defaultStorageSuffix)
@@ -142,15 +141,15 @@ public static void AddHiveMetastoreToConfigurations(AzureHDInsightMetastore hive
             string connectionUrl =
                 string.Format(Constants.MetastoreConfigurations.ConnectionUrlFormat, hiveMetastore.SqlAzureServerName, hiveMetastore.DatabaseName);
 
-            configurations.AddOrCombineConfigurations(ConfigurationKey.HiveSite, new Dictionary<string, string>
+            configurations.AddOrCombineConfigurations(Constants.ConfigurationKey.HiveSite, new Dictionary<string, string>
                 {
                     {Constants.MetastoreConfigurations.HiveSite.ConnectionUrlKey, connectionUrl},
                     {Constants.MetastoreConfigurations.HiveSite.ConnectionUserNameKey, hiveMetastore.Credential.UserName},
                     {Constants.MetastoreConfigurations.HiveSite.ConnectionPasswordKey, hiveMetastore.Credential.Password.ConvertToString()},
                     {Constants.MetastoreConfigurations.HiveSite.ConnectionDriverNameKey, Constants.MetastoreConfigurations.HiveSite.ConnectionDriverNameValue}
                 });
 
-            configurations.AddOrCombineConfigurations(ConfigurationKey.HiveEnv, new Dictionary<string, string>
+            configurations.AddOrCombineConfigurations(Constants.ConfigurationKey.HiveEnv, new Dictionary<string, string>
                 {
                     {Constants.MetastoreConfigurations.HiveEnv.DatabaseKey, Constants.MetastoreConfigurations.DatabaseValue},
                     {Constants.MetastoreConfigurations.HiveEnv.DatabaseNameKey, hiveMetastore.DatabaseName},
@@ -169,7 +168,7 @@ public static void AddOozieMetastoreToConfigurations(AzureHDInsightMetastore ooz
             }
             string connectionUrl = string.Format(Constants.MetastoreConfigurations.ConnectionUrlFormat, oozieMetastore.SqlAzureServerName, oozieMetastore.DatabaseName);
 
-            configurations.AddOrCombineConfigurations(ConfigurationKey.OozieSite, new Dictionary<string, string>
+            configurations.AddOrCombineConfigurations(Constants.ConfigurationKey.OozieSite, new Dictionary<string, string>
                 {
                     {Constants.MetastoreConfigurations.OozieSite.UrlKey, connectionUrl},
                     {Constants.MetastoreConfigurations.OozieSite.UserNameKey, oozieMetastore.Credential.UserName},
@@ -178,7 +177,7 @@ public static void AddOozieMetastoreToConfigurations(AzureHDInsightMetastore ooz
                     {Constants.MetastoreConfigurations.OozieSite.SchemaKey, Constants.MetastoreConfigurations.OozieSite.SchemaValue}
                 });
 
-            configurations.AddOrCombineConfigurations(ConfigurationKey.OozieEnv, new Dictionary<string, string>
+            configurations.AddOrCombineConfigurations(Constants.ConfigurationKey.OozieEnv, new Dictionary<string, string>
                 {
                     {Constants.MetastoreConfigurations.OozieEnv.DatabaseKey, Constants.MetastoreConfigurations.DatabaseValue},
                     {Constants.MetastoreConfigurations.OozieEnv.DatabaseNameKey, oozieMetastore.DatabaseName},
@@ -189,6 +188,22 @@ public static void AddOozieMetastoreToConfigurations(AzureHDInsightMetastore ooz
                 });
         }
 
+        public static void AddCustomAmbariDatabaseToConfigurations(AzureHDInsightMetastore ambariDatabase, IDictionary<string, Dictionary<string, string>> configurations)
+        {
+            if (Uri.CheckHostName(ambariDatabase.SqlAzureServerName) != UriHostNameType.Dns)
+            {
+                throw new ArgumentException("Please provide the fully qualified sql server name.");
+            }
+
+            configurations.AddOrCombineConfigurations(Constants.ConfigurationKey.AmbariConf, new Dictionary<string, string>
+                {
+                    {Constants.AmbariConfiguration.SqlServerKey, ambariDatabase.SqlAzureServerName},
+                    {Constants.AmbariConfiguration.DatabaseNameKey, ambariDatabase.DatabaseName},
+                    {Constants.AmbariConfiguration.DatabaseUserKey, ambariDatabase.Credential.UserName},
+                    {Constants.AmbariConfiguration.DatabasePasswordKey, ambariDatabase.Credential.Password.ConvertToString()}
+                });
+        }
+
         public static VirtualNetworkProfile CreateVirtualNetworkProfile(string virtualNetworkId, string subnetName)
         {
             if (string.IsNullOrEmpty(virtualNetworkId) && string.IsNullOrEmpty(subnetName))
diff --git a/src/HDInsight/HDInsight/Constants.cs b/src/HDInsight/HDInsight/Constants.cs
@@ -143,5 +143,116 @@ public static class Errors
             public static string ERROR_INPUT_CANNOT_BE_EMPTY = "Input cannot be empty";
             public static string ERROR_SCHEME_SPECIFIED_IN_STORAGE_FQDN = "Please specify fully qualified storage endpoint without the scheme";
         }
+
+        public static class ConfigurationKey
+        {
+            /// <summary>
+            /// The constant for Core site configs.
+            /// </summary>
+            public const string CoreSite = "core-site";
+
+            /// <summary>
+            /// The constant for Hive site configs.
+            /// </summary>
+            public const string HiveSite = "hive-site";
+
+            /// <summary>
+            /// The constant for hive environment configs.
+            /// </summary>
+            public const string HiveEnv = "hive-env";
+
+            /// <summary>
+            /// The constant for Oozie site configs.
+            /// </summary>
+            public const string OozieSite = "oozie-site";
+
+            /// <summary>
+            /// The constant for Oozie environment configs.
+            /// </summary>
+            public const string OozieEnv = "oozie-env";
+
+            /// <summary>
+            /// The constant for WebHCAT site configs.
+            /// </summary>
+            public const string WebHCatSite = "webhcat-site";
+
+            /// <summary>
+            /// The constant for HBase environment configs.
+            /// </summary>
+            public const string HBaseEnv = "hbase-env";
+
+            /// <summary>
+            /// The constant for HBase site configs.
+            /// </summary>
+            public const string HBaseSite = "hbase-site";
+
+            /// <summary>
+            /// The constant for Storm site configs.
+            /// </summary>
+            public const string StormSite = "storm-site";
+
+            /// <summary>
+            /// The constant for Yarn site configs.
+            /// </summary>
+            public const string YarnSite = "yarn-site";
+
+            /// <summary>
+            /// The constant for MapRed site configs.
+            /// </summary>
+            public const string MapRedSite = "mapred-site";
+
+            /// <summary>
+            /// The constant for Tez site configs.
+            /// </summary>
+            public const string TezSite = "tez-site";
+
+            /// <summary>
+            /// The constant for HDFS site configs.
+            /// </summary>
+            public const string HdfsSite = "hdfs-site";
+
+            /// <summary>
+            /// The constant for Gateway configs.
+            /// </summary>
+            public const string Gateway = "gateway";
+
+            /// <summary>
+            /// The constant for cluster identity configs.
+            /// </summary>
+            public const string ClusterIdentity = "clusterIdentity";
+
+            /// <summary>
+            /// The constant for Spark-Defaults configs.
+            /// </summary>
+            public const string SparkDefaults = "spark-defaults";
+
+            /// <summary>
+            /// The constant for Spark-Thrift-SparkConf configs.
+            /// </summary>
+            public const string SparkThriftConf = "spark-thrift-sparkconf";
+
+            /// <summary>
+            /// The constant for Spark2-Defaults configs.
+            /// </summary>
+            public const string Spark2Defaults = "spark2-defaults";
+
+            /// <summary>
+            /// The constant for Spark2-Thrift-SparkConf configs.
+            /// </summary>
+            public const string Spark2ThriftConf = "spark2-thrift-sparkconf";
+
+            /// <summary>
+            /// The constant for custom ambari db configs.
+            /// </summary>
+            public const string AmbariConf = "ambari-conf";
+        }
+
+        public static class AmbariConfiguration
+        {
+            public const string SqlServerKey = "database-server";
+            public const string DatabaseNameKey = "database-name";
+            public const string DatabaseUserKey = "database-user-name";
+            public const string DatabasePasswordKey = "database-user-password";
+        }
     }
 }
diff --git a/src/HDInsight/HDInsight/ManagementCommands/AddAzureHDInsightMetastoreCommand.cs b/src/HDInsight/HDInsight/ManagementCommands/AddAzureHDInsightMetastoreCommand.cs
@@ -80,6 +80,9 @@ public override void ExecuteCmdlet()
                 case AzureHDInsightMetastoreType.OozieMetastore:
                     Config.OozieMetastore = _metastore;
                     break;
+                case AzureHDInsightMetastoreType.AmbariDatabase:
+                    Config.AmbariDatabase = _metastore;
+                    break;
             }
 
             WriteObject(Config);
diff --git a/src/HDInsight/HDInsight/ManagementCommands/NewAzureHDInsightClusterCommand.cs b/src/HDInsight/HDInsight/ManagementCommands/NewAzureHDInsightClusterCommand.cs
@@ -114,6 +114,7 @@ public AzureHDInsightConfig Config
                     ZookeeperNodeSize = ZookeeperNodeSize,
                     HiveMetastore = HiveMetastore,
                     OozieMetastore = OozieMetastore,
+                    AmbariDatabase = AmbariDatabase,
                     ObjectId = ObjectId,
                     ApplicationId = ApplicationId,
                     AADTenantId = AadTenantId,
@@ -175,6 +176,7 @@ var storageAccount in
                 ZookeeperNodeSize = value.ZookeeperNodeSize;
                 HiveMetastore = value.HiveMetastore;
                 OozieMetastore = value.OozieMetastore;
+                AmbariDatabase = value.AmbariDatabase;
                 CertificateFileContents = value.CertificateFileContents;
                 CertificateFilePath = value.CertificateFilePath;
                 AadTenantId = value.AADTenantId;
@@ -220,6 +222,9 @@ var storageAccount in
         [Parameter(HelpMessage = "Gets or sets the database to store the metadata for Hive.")]
         public AzureHDInsightMetastore HiveMetastore { get; set; }
 
+        [Parameter(HelpMessage = "Gets or sets the database for ambari.")]
+        public AzureHDInsightMetastore AmbariDatabase { get; set; }
+
         [Parameter(HelpMessage = "Gets additional Azure Storage Account that you want to enable access to.")]
         public Dictionary<string, string> AdditionalStorageAccounts { get; private set; }
 
@@ -447,6 +452,12 @@ var storageAccount in
                 ClusterCreateHelper.AddHiveMetastoreToConfigurations(HiveMetastore, clusterConfigurations);
             }
 
+            // Handle Custom Ambari Database
+            if (AmbariDatabase != null)
+            {
+                ClusterCreateHelper.AddCustomAmbariDatabaseToConfigurations(AmbariDatabase, clusterConfigurations);
+            }
+
             // Handle ADLSGen1 identity
             if (!string.IsNullOrEmpty(CertificatePassword))
             {
@@ -553,7 +564,6 @@ var storageAccount in
                 networkProperties = new NetworkProperties(ResourceProviderConnection, PrivateLink);
             }
 
-
             // Construct cluster create parameter
             ClusterCreateParametersExtended createParams = new ClusterCreateParametersExtended
             {
diff --git a/src/HDInsight/HDInsight/Models/Management/AzureHDInsightConfig.cs b/src/HDInsight/HDInsight/Models/Management/AzureHDInsightConfig.cs
@@ -87,6 +87,11 @@ public class AzureHDInsightConfig
         /// </summary>
         public AzureHDInsightMetastore HiveMetastore { get; set; }
 
+        /// <summary>
+        /// Gets or sets the database for ambari.
+        /// </summary>
+        public AzureHDInsightMetastore AmbariDatabase { get; set; }
+
         /// <summary>
         /// Gets Object id of the service principal. 
         /// </summary>
diff --git a/src/HDInsight/HDInsight/Models/Management/AzureHDInsightMetastoreType.cs b/src/HDInsight/HDInsight/Models/Management/AzureHDInsightMetastoreType.cs
@@ -27,6 +27,11 @@ public enum AzureHDInsightMetastoreType
         /// <summary>
         /// An Oozie metastore.
         /// </summary>
-        OozieMetastore
+        OozieMetastore,
+
+        /// <summary>
+        /// An Ambari database
+        /// </summary>
+        AmbariDatabase
     }
 }
diff --git a/src/HDInsight/HDInsight/help/Add-AzHDInsightMetastore.md b/src/HDInsight/HDInsight/help/Add-AzHDInsightMetastore.md
diff --git a/src/HDInsight/HDInsight/help/New-AzHDInsightCluster.md b/src/HDInsight/HDInsight/help/New-AzHDInsightCluster.md
diff --git a/tools/SecurityTools/CredScanSuppressions.json b/tools/SecurityTools/CredScanSuppressions.json

Original file line number	Diff line number	Diff line change
`@@ -84,5 +84,12 @@ public void TestCreateClusterWithRelayOutoundAndPrivateLink()`
`84`	`84`	`{`
`85`	`85`	`TestController.NewInstance.RunPowerShellTest(_logger, "Test-CreateClusterWithRelayOutoundAndPrivateLink");`
`86`	`86`	`}`
	`87`	`+`
	`88`	`+ [Fact]`
	`89`	`+ [Trait(Category.AcceptanceType, Category.CheckIn)]`
	`90`	`+ public void TestCreateClusterWithCustomAmbariDatabase()`
	`91`	`+ {`
	`92`	`+ TestController.NewInstance.RunPowerShellTest(_logger, "Test-CreateClusterWithCustomAmbariDatabase");`
	`93`	`+ }`
`87`	`94`	`}`
`88`	`95`	`}`
Original file line number	Diff line number	Diff line change
`@@ -80,6 +80,9 @@ public override void ExecuteCmdlet()`
`80`	`80`	`case AzureHDInsightMetastoreType.OozieMetastore:`
`81`	`81`	`Config.OozieMetastore = _metastore;`
`82`	`82`	`break;`
	`83`	`+ case AzureHDInsightMetastoreType.AmbariDatabase:`
	`84`	`+ Config.AmbariDatabase = _metastore;`
	`85`	`+ break;`
`83`	`86`	`}`
`84`	`87`
`85`	`88`	`WriteObject(Config);`
Original file line number	Diff line number	Diff line change
`@@ -27,6 +27,11 @@ public enum AzureHDInsightMetastoreType`
`27`	`27`	`/// <summary>`
`28`	`28`	`/// An Oozie metastore.`
`29`	`29`	`/// </summary>`
`30`		`- OozieMetastore`
	`30`	`+ OozieMetastore,`
	`31`	`+`
	`32`	`+ /// <summary>`
	`33`	`+ /// An Ambari database`
	`34`	`+ /// </summary>`
	`35`	`+ AmbariDatabase`
`31`	`36`	`}`
`32`	`37`	`}`