Azure · wyunchi-ms · Oct 16, 2020 · Sep 18, 2020 · Oct 15, 2020 · Oct 15, 2020
diff --git a/src/HDInsight/HDInsight.Test/DataLakeStoreDefaultFilesystemTests.cs b/src/HDInsight/HDInsight.Test/DataLakeStoreDefaultFilesystemTests.cs
@@ -29,7 +29,7 @@ namespace Commands.HDInsight.Test.UnitTests
     public class DataLakeStoreDefaultFilesystemTests : HDInsightTestBase
     {
         private NewAzureHDInsightClusterCommand cmdlet;
-        private const string StorageName = "dummystorage.azuredatalakestore.net";
+        private const string StorageAccountResourceId = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/fakerg/providers/Microsoft.Storage/storageAccounts/dummystorage";
         private const int ClusterSize = 4;
         private Guid ObjectId = new Guid("11111111-1111-1111-1111-111111111111");
         private Guid ApplicationId = new Guid("11111111-1111-1111-1111-111111111111");
@@ -66,7 +66,7 @@ public void CanCreateClusterConfigWithDataLakeStoreParameters()
                 CertificateFilePath = Certificate,
                 AadTenantId = AadTenantId,
                 CertificatePassword = CertificatePassword,
-                DefaultStorageAccountName = StorageName
+                StorageAccountResourceId = StorageAccountResourceId
             };
 
             newclusteridentitycmdlet.ExecuteCmdlet();
@@ -80,7 +80,7 @@ public void CanCreateClusterConfigWithDataLakeStoreParameters()
                                 c.ObjectId == ObjectId &&
                                 c.ApplicationId == ApplicationId &&
                                 c.CertificateFilePath == Certificate &&
-                                c.DefaultStorageAccountName == StorageName
+                                c.StorageAccountResourceId == StorageAccountResourceId
                                 )),
                 Times.Once);
         }
@@ -98,7 +98,7 @@ public void CanCreateDataLakeClusterWithCertificateFileContents()
                 CertificateFileContents = CertificateFileContents,
                 AadTenantId = AadTenantId,
                 CertificatePassword = CertificatePassword,
-                DefaultStorageAccountName = StorageName
+                StorageAccountResourceId = StorageAccountResourceId
             };
 
             clusterIdentityCmdlet.ExecuteCmdlet();
@@ -112,7 +112,7 @@ public void CanCreateDataLakeClusterWithCertificateFileContents()
                                 c.ObjectId == ObjectId &&
                                 c.ApplicationId == ApplicationId &&
                                 c.CertificateFileContents == CertificateFileContents &&
-                                c.DefaultStorageAccountName == StorageName
+                                c.StorageAccountResourceId == StorageAccountResourceId
                                 )),
                 Times.Once);
         }

diff --git a/src/HDInsight/HDInsight.Test/ScenarioTests/Common.ps1 b/src/HDInsight/HDInsight.Test/ScenarioTests/Common.ps1
@@ -73,7 +73,7 @@ class ClusterCommonCreateParameter{
       [string] $clusterName
       [string] $location
       [string] $resourceGroupName
-      [string] $storageAccountName
+      [string] $storageAccountResourceId
       [string] $clusterType
       [int] $clusterSizeInNodes
       [string] $storageAccountKey
@@ -82,13 +82,13 @@ class ClusterCommonCreateParameter{
       [string] $minSupportedTlsVersion
 
 	  ClusterCommonCreateParameter([string] $clusterName, [string] $location, [string] $resourceGroupName,
-                                   [string] $storageAccountName, [string] $clusterType, [int] $clusterSizeInNodes, 
+                                   [string] $storageAccountResourceId, [string] $clusterType, [int] $clusterSizeInNodes, 
                                    [string] $storageAccountKey, [System.Management.Automation.PSCredential] $httpCredential,
                                    [System.Management.Automation.PSCredential] $sshCredential, [string] $minSupportedTlsVersion){
                 $this.clusterName=$clusterName
                 $this.location=$location
                 $this.resourceGroupName=$resourceGroupName
-                $this.storageAccountName=$storageAccountName
+                $this.storageAccountResourceId=$storageAccountResourceId
                 $this.clusterType=$clusterType
                 $this.clusterSizeInNodes=$clusterSizeInNodes
                 $this.storageAccountKey=$storageAccountKey
@@ -100,9 +100,9 @@ class ClusterCommonCreateParameter{
 
 <#
 .SYNOPSIS
- Create Common Parameter for creating cluster.
+ Create Common Parameter with WASB for creating cluster.
 #>
-function Prepare-ClusterCreateParameterForWASB{
+function Prepare-ClusterCreateParameter{
     param(
       [string] $clusterName="hdi-ps-test",
       [string] $location="East US",
@@ -118,8 +118,9 @@ function Prepare-ClusterCreateParameterForWASB{
 
     $storageAccountName=Generate-StorageAccountName($storageAccountName)
 
-    $storageAccount= New-AzStorageAccount -ResourceGroupName $resourceGroupName -Location $location -Name $storageAccountName -TypeString "Standard_RAGRS"
+	$storageAccount= New-AzStorageAccount -ResourceGroupName $resourceGroupName -Location $location -Name $storageAccountName -TypeString Standard_RAGRS
 
+	$storageAccountResourceId=$storageAccount.Id
     $storageAccountKey=Get-AzStorageAccountKey -ResourceGroupName $resourceGroupName -Name $storageAccountName
     $storageAccountKey=$storageAccountKey[0].Value
 
@@ -131,9 +132,9 @@ function Prepare-ClusterCreateParameterForWASB{
     $httpCredential=New-Object System.Management.Automation.PSCredential($httpUser, $httpPassword)
     $sshCredential=New-Object System.Management.Automation.PSCredential($sshUser, $sshPassword)
 
-    $clusterSizeInNodes=2
+    $clusterSizeInNodes=3
     $minSupportedTlsVersion="1.2"
-    return [ClusterCommonCreateParameter]::new($clusterName, $location,  $resourceGroupName,$storageAccountName, 
+    return [ClusterCommonCreateParameter]::new($clusterName, $location,  $resourceGroupName,$storageAccountResourceId, 
                                                $clusterType, $clusterSizeInNodes,$storageAccountKey, $httpCredential,
                                                $sshCredential, $minSupportedTlsVersion)
 }
@@ -151,7 +152,7 @@ function Create-CMKCluster{
 	  [string] $keyName="key-ps-cmktest"
     )
 
-    $params=Prepare-ClusterCreateParameterForWASB -clusterName $clusterName -location $location 
+    $params=Prepare-ClusterCreateParameter -clusterName $clusterName -location $location 
 
     # new user-assigned identity
     $assignedIdentity= New-AzUserAssignedIdentity -ResourceGroupName $params.resourceGroupName -Name $assignedIdentityName
@@ -169,8 +170,8 @@ function Create-CMKCluster{
     $encryptionKeyName=$encryptionKey.Name
     # new hdi cluster with cmk
     $cluster=New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName -ClusterName $params.clusterName `
-    -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType -DefaultStorageAccountName $params.storageAccountName `
-    -DefaultStorageAccountKey $params.storageAccountKey -HttpCredential $params.httpCredential -SshCredential $params.sshCredential  `
+    -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType -StorageAccountResourceId $params.storageAccountResourceId `
+    -StorageAccountKey $params.storageAccountKey -HttpCredential $params.httpCredential -SshCredential $params.sshCredential  `
     -AssignedIdentity $assignedIdentityId -EncryptionKeyName $encryptionKeyName -EncryptionKeyVersion $encryptionKeyVersion `
     -EncryptionVaultUri $encryptionVaultUri
 

diff --git a/src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightAutoscaleTests.ps1 b/src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightAutoscaleTests.ps1
@@ -22,12 +22,12 @@ function Test-AutoscaleRelatedCommands{
 	try
 	{
 		# prepare parameter for creating parameter
-		$params= Prepare-ClusterCreateParameterForWASB
+		$params= Prepare-ClusterCreateParameter
 
 		# test create cluster
 		$cluster = New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
 		-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
-		-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
+		-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
 		-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
 		-MinSupportedTlsVersion $params.minSupportedTlsVersion
 

diff --git a/src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightClusterTests.cs b/src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightClusterTests.cs
@@ -50,13 +50,6 @@ public void TestCreateClusterWithEncryptionInTransit()
             TestController.NewInstance.RunPowerShellTest(_logger, "Test-CreateClusterWithEncryptionInTransit");
         }
 
-        [Fact]
-        [Trait(Category.AcceptanceType, Category.CheckIn)]
-        public void TestCreateClusterWithPrivateLink()
-        {
-            TestController.NewInstance.RunPowerShellTest(_logger, "Test-CreateClusterWithPrivateLink");
-        }
-
         [Fact]
         [Trait(Category.AcceptanceType, Category.CheckIn)]
         public void TestCreateClusterWithEncryptionAtHost()
@@ -77,5 +70,12 @@ public void TestCreateClusterWithScheduleBasedAutoscale()
         {
             TestController.NewInstance.RunPowerShellTest(_logger, "Test-CreateClusterWithScheduleBasedAutoscale");
         }
+
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void TestCreateClusterWithKafkaRestProxy()
+        {
+            TestController.NewInstance.RunPowerShellTest(_logger, "Test-CreateClusterWithKafkaRestProxy");
+        }
     }
 }
diff --git a/src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightClusterTests.ps1 b/src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightClusterTests.ps1
@@ -23,12 +23,12 @@ function Test-ClusterRelatedCommands{
 	try
 	{
 		# prepare parameter for creating parameter
-		$params= Prepare-ClusterCreateParameterForWASB
+		$params= Prepare-ClusterCreateParameter
 
 		# test create cluster
 		$cluster = New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
 		-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
-		-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
+		-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
 		-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
 		-MinSupportedTlsVersion $params.minSupportedTlsVersion
 
@@ -104,13 +104,13 @@ function Test-CreateClusterWithEncryptionInTransit{
 	try
 	{
 		# prepare parameter for creating parameter
-		$params= Prepare-ClusterCreateParameterForWASB -Location "South Central US"
+		$params= Prepare-ClusterCreateParameter -Location "South Central US"
 		$encryptionInTransit=$true
 
 		# create cluster
 		$cluster=New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
 		-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
-		-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
+		-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
 		-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
 		-MinSupportedTlsVersion $params.minSupportedTlsVersion -EncryptionInTransit $encryptionInTransit
 
@@ -125,45 +125,6 @@ function Test-CreateClusterWithEncryptionInTransit{
 	}
 }
 
-<#
-.SYNOPSIS
-Test Create Azure HDInsight Cluster which Private Link
-#>
-
-function Test-CreateClusterWithPrivateLink{
-
-	# Create some resources that will be used throughout test
-	try
-	{
-		# prepare parameter for creating parameter
-		$params= Prepare-ClusterCreateParameterForWASB -location "South Central US"
-
-		# Prepare virtual network
-		$vnetName=Generate-Name("hdi-ps-vnet")
-		$vnet=Create-VnetkWithSubnet -location $params.location -resourceGroupName $params.resourceGroupName `
-		-vnetName $vnetName -subnetPrivateLinkServiceNetworkPoliciesFlag $false 
-
-		# create cluster
-		$cluster=New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
-		-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
-		-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
-		-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
-		-MinSupportedTlsVersion $params.minSupportedTlsVersion `
-		-VirtualNetworkId $vnet.Id -SubnetName $vnet.Subnets[0].Id `
-		-PublicNetworkAccessType OutboundOnly -OutboundPublicNetworkAccessType PublicLoadBalancer
-
-		Assert-AreEqual $cluster.PublicNetworkAccessType OutboundOnly
-		Assert-AreEqual $cluster.OutboundPublicNetworkAccessType PublicLoadBalancer
-
-	}
-	finally
-	{
-		# Delete cluster and resource group
-		Remove-AzHDInsightCluster -ClusterName $cluster.Name
-		Remove-AzResourceGroup -ResourceGroupName $cluster.ResourceGroup
-	}
-}
-
 <#
 .SYNOPSIS
 Test Create Azure HDInsight Cluster which enalbes Encryption At Host
@@ -175,7 +136,7 @@ function Test-CreateClusterWithEncryptionAtHost{
 	try
 	{
 		# prepare parameter for creating parameter
-		$params= Prepare-ClusterCreateParameterForWASB -location "South Central US"
+		$params= Prepare-ClusterCreateParameter -location "South Central US"
 		$encryptionAtHost=$true
 		$workerNodeSize="Standard_DS14_v2"
 		$headNodeSize="Standard_DS14_v2"
@@ -185,7 +146,7 @@ function Test-CreateClusterWithEncryptionAtHost{
 		$cluster=New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
 		-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
 		-WorkerNodeSize $workerNodeSize -HeadNodeSize $headNodeSize -ZookeeperNodeSize $zookeeperNodeSize `
-		-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
+		-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
 		-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
 		-MinSupportedTlsVersion $params.minSupportedTlsVersion -EncryptionAtHost $encryptionAtHost
 
@@ -211,15 +172,15 @@ function Test-CreateClusterWithLoadBasedAutoscale{
 	try
 	{
 		# prepare parameter for creating parameter
-		$params= Prepare-ClusterCreateParameterForWASB -location "East US"
+		$params= Prepare-ClusterCreateParameter -location "East US"
 
 		# create autoscale cofiguration
 		$autoscaleConfiguration=New-AzHDInsightClusterAutoscaleConfiguration -MinWorkerNodeCount 4 -MaxWorkerNodeCount 5
 
 		# create cluster with load-based autoscale
 		$cluster=New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
 		-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
-		-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
+		-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
 		-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
 		-MinSupportedTlsVersion $params.minSupportedTlsVersion -Version 4.0 `
 		-AutoscaleConfiguration $autoscaleConfiguration
@@ -247,7 +208,7 @@ function Test-CreateClusterWithScheduleBasedAutoscale{
 	try
 	{
 		# prepare parameter for creating parameter
-		$params= Prepare-ClusterCreateParameterForWASB -location "East US"
+		$params= Prepare-ClusterCreateParameter -location "East US"
 
 		# create autoscale schedule condition
 		$condition1=New-AzHDInsightClusterAutoscaleScheduleCondition -Time "09:00" -WorkerNodeCount 4 -Day Monday,Tuesday
@@ -260,7 +221,7 @@ function Test-CreateClusterWithScheduleBasedAutoscale{
 		# create cluster with schedule-based autoscale
 		$cluster=New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
 		-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
-		-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
+		-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
 		-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
 		-MinSupportedTlsVersion $params.minSupportedTlsVersion -Version 4.0 `
 		-AutoscaleConfiguration $autoscaleConfiguration
@@ -277,3 +238,40 @@ function Test-CreateClusterWithScheduleBasedAutoscale{
 		Remove-AzResourceGroup -ResourceGroupName $cluster.ResourceGroup
 	}
 }
+
+<#
+.SYNOPSIS
+Test Create Azure HDInsight Cluster with kafka rest proxy.
+#>
+
+function Test-CreateClusterWithKafkaRestProxy{
+# Create some resources that will be used throughout test
+	try
+	{
+		# prepare parameter for creating parameter
+		$params= Prepare-ClusterCreateParameter -location "South Central US" -clusterType Kafka
+		$kafkaClientGroupName="FakeClientGroup"
+		$kafkaClientGroupId="00000000-0000-0000-0000-000000000000"
+		$disksPerWorkerNode=2
+
+		# test create cluster
+		$cluster = New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
+		-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
+		-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
+		-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
+		-MinSupportedTlsVersion $params.minSupportedTlsVersion -KafkaClientGroupId  $kafkaClientGroupId `
+		-KafkaClientGroupName $kafkaClientGroupName -DisksPerWorkerNode $disksPerWorkerNode `
+		-KafkaManagementNodeSize Standard_D4_v2
+
+		Assert-NotNull $cluster
+		#test Get-AzHDInsightCluster
+		$resultCluster = Get-AzHDInsightCluster -ClusterName $cluster.Name
+		Assert-AreEqual $resultCluster.Name  $cluster.Name
+	}
+	finally
+	{
+		# Delete cluster and resource group
+		Remove-AzHDInsightCluster -ClusterName $cluster.Name
+		Remove-AzResourceGroup -ResourceGroupName $cluster.ResourceGroup
+	}
+}
diff --git a/src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightHostTests.ps1 b/src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightHostTests.ps1
@@ -23,12 +23,12 @@ function Test-HostRelatedCommands{
 	try
 	{
 		# prepare parameter for creating parameter
-		$params= Prepare-ClusterCreateParameterForWASB
+		$params= Prepare-ClusterCreateParameter
 
 		# create cluster that will be used throughout test
 		$cluster = New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
 		-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
-		-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
+		-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
 		-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
 		-MinSupportedTlsVersion $params.minSupportedTlsVersion
 		Assert-NotNull $cluster

diff --git a/src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightSetGatewayCredentialTest.ps1 b/src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightSetGatewayCredentialTest.ps1
@@ -22,12 +22,12 @@ function Test-SetGatewayCredential{
 	try
 	{
 		# prepare parameter for creating parameter
-		$params= Prepare-ClusterCreateParameterForWASB
+		$params= Prepare-ClusterCreateParameter
 
 		# create cluster that will be used throughout test
 		$cluster = New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
 		-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
-		-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
+		-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
 		-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
 		-MinSupportedTlsVersion $params.minSupportedTlsVersion