Skip to content

Commit 37b5794

Browse files
author
Zhenyu Zhou
committed
1. Support ADLSGen2 MSI: Remove the Default Prefix of Storage parameter,
add StorageFileSystem, StorageAccountManagedIdentity parameters 2. Support KafkaRestProxy: add new parameters: KafkaClientGroupName, KafkaClientGroupId, KafkaManagementNodeSize 3. Support ESP and HIB: Change Domin to DominResourceId and -EnableIDBroker switch parameter 4. Require Storage Resource Id instead name
1 parent 3ec5cf4 commit 37b5794

File tree

43 files changed

+4264
-760
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+4264
-760
lines changed

‎src/HDInsight/HDInsight.Test/DataLakeStoreDefaultFilesystemTests.cs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ namespace Commands.HDInsight.Test.UnitTests
2929
public class DataLakeStoreDefaultFilesystemTests : HDInsightTestBase
3030
{
3131
private NewAzureHDInsightClusterCommand cmdlet;
32-
private const string StorageName = "dummystorage.azuredatalakestore.net";
32+
private const string StorageAccountResourceId = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/fakerg/providers/Microsoft.Storage/storageAccounts/dummystorage";
3333
private const int ClusterSize = 4;
3434
private Guid ObjectId = new Guid("11111111-1111-1111-1111-111111111111");
3535
private Guid ApplicationId = new Guid("11111111-1111-1111-1111-111111111111");
@@ -66,7 +66,7 @@ public void CanCreateClusterConfigWithDataLakeStoreParameters()
6666
CertificateFilePath = Certificate,
6767
AadTenantId = AadTenantId,
6868
CertificatePassword = CertificatePassword,
69-
DefaultStorageAccountName = StorageName
69+
StorageAccountResourceId = StorageAccountResourceId
7070
};
7171

7272
newclusteridentitycmdlet.ExecuteCmdlet();
@@ -80,7 +80,7 @@ public void CanCreateClusterConfigWithDataLakeStoreParameters()
8080
c.ObjectId == ObjectId &&
8181
c.ApplicationId == ApplicationId &&
8282
c.CertificateFilePath == Certificate &&
83-
c.DefaultStorageAccountName == StorageName
83+
c.StorageAccountResourceId == StorageAccountResourceId
8484
)),
8585
Times.Once);
8686
}
@@ -98,7 +98,7 @@ public void CanCreateDataLakeClusterWithCertificateFileContents()
9898
CertificateFileContents = CertificateFileContents,
9999
AadTenantId = AadTenantId,
100100
CertificatePassword = CertificatePassword,
101-
DefaultStorageAccountName = StorageName
101+
StorageAccountResourceId = StorageAccountResourceId
102102
};
103103

104104
clusterIdentityCmdlet.ExecuteCmdlet();
@@ -112,7 +112,7 @@ public void CanCreateDataLakeClusterWithCertificateFileContents()
112112
c.ObjectId == ObjectId &&
113113
c.ApplicationId == ApplicationId &&
114114
c.CertificateFileContents == CertificateFileContents &&
115-
c.DefaultStorageAccountName == StorageName
115+
c.StorageAccountResourceId == StorageAccountResourceId
116116
)),
117117
Times.Once);
118118
}

‎src/HDInsight/HDInsight.Test/ScenarioTests/Common.ps1‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ class ClusterCommonCreateParameter{
7373
[string] $clusterName
7474
[string] $location
7575
[string] $resourceGroupName
76-
[string] $storageAccountName
76+
[string] $storageAccountResourceId
7777
[string] $clusterType
7878
[int] $clusterSizeInNodes
7979
[string] $storageAccountKey
@@ -82,13 +82,13 @@ class ClusterCommonCreateParameter{
8282
[string] $minSupportedTlsVersion
8383

8484
ClusterCommonCreateParameter([string] $clusterName, [string] $location, [string] $resourceGroupName,
85-
[string] $storageAccountName, [string] $clusterType, [int] $clusterSizeInNodes,
85+
[string] $storageAccountResourceId, [string] $clusterType, [int] $clusterSizeInNodes,
8686
[string] $storageAccountKey, [System.Management.Automation.PSCredential] $httpCredential,
8787
[System.Management.Automation.PSCredential] $sshCredential, [string] $minSupportedTlsVersion){
8888
$this.clusterName=$clusterName
8989
$this.location=$location
9090
$this.resourceGroupName=$resourceGroupName
91-
$this.storageAccountName=$storageAccountName
91+
$this.storageAccountResourceId=$storageAccountResourceId
9292
$this.clusterType=$clusterType
9393
$this.clusterSizeInNodes=$clusterSizeInNodes
9494
$this.storageAccountKey=$storageAccountKey
@@ -100,9 +100,9 @@ class ClusterCommonCreateParameter{
100100

101101
<#
102102
.SYNOPSIS
103-
Create Common Parameter for creating cluster.
103+
Create Common Parameter with WASB for creating cluster.
104104
#>
105-
function Prepare-ClusterCreateParameterForWASB{
105+
function Prepare-ClusterCreateParameter{
106106
param(
107107
[string] $clusterName="hdi-ps-test",
108108
[string] $location="East US",
@@ -118,8 +118,9 @@ function Prepare-ClusterCreateParameterForWASB{
118118

119119
$storageAccountName=Generate-StorageAccountName($storageAccountName)
120120

121-
$storageAccount= New-AzStorageAccount -ResourceGroupName $resourceGroupName -Location $location -Name $storageAccountName -TypeString "Standard_RAGRS"
121+
$storageAccount= New-AzStorageAccount -ResourceGroupName $resourceGroupName -Location $location -Name $storageAccountName -TypeString Standard_RAGRS
122122

123+
$storageAccountResourceId=$storageAccount.Id
123124
$storageAccountKey=Get-AzStorageAccountKey -ResourceGroupName $resourceGroupName -Name $storageAccountName
124125
$storageAccountKey=$storageAccountKey[0].Value
125126

@@ -131,9 +132,9 @@ function Prepare-ClusterCreateParameterForWASB{
131132
$httpCredential=New-Object System.Management.Automation.PSCredential($httpUser, $httpPassword)
132133
$sshCredential=New-Object System.Management.Automation.PSCredential($sshUser, $sshPassword)
133134

134-
$clusterSizeInNodes=2
135+
$clusterSizeInNodes=3
135136
$minSupportedTlsVersion="1.2"
136-
return [ClusterCommonCreateParameter]::new($clusterName, $location, $resourceGroupName,$storageAccountName,
137+
return [ClusterCommonCreateParameter]::new($clusterName, $location, $resourceGroupName,$storageAccountResourceId,
137138
$clusterType, $clusterSizeInNodes,$storageAccountKey, $httpCredential,
138139
$sshCredential, $minSupportedTlsVersion)
139140
}
@@ -151,7 +152,7 @@ function Create-CMKCluster{
151152
[string] $keyName="key-ps-cmktest"
152153
)
153154

154-
$params=Prepare-ClusterCreateParameterForWASB -clusterName $clusterName -location $location
155+
$params=Prepare-ClusterCreateParameter -clusterName $clusterName -location $location
155156

156157
# new user-assigned identity
157158
$assignedIdentity= New-AzUserAssignedIdentity -ResourceGroupName $params.resourceGroupName -Name $assignedIdentityName
@@ -169,8 +170,8 @@ function Create-CMKCluster{
169170
$encryptionKeyName=$encryptionKey.Name
170171
# new hdi cluster with cmk
171172
$cluster=New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName -ClusterName $params.clusterName `
172-
-ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType -DefaultStorageAccountName $params.storageAccountName `
173-
-DefaultStorageAccountKey $params.storageAccountKey -HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
173+
-ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType -StorageAccountResourceId $params.storageAccountResourceId `
174+
-StorageAccountKey $params.storageAccountKey -HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
174175
-AssignedIdentity $assignedIdentityId -EncryptionKeyName $encryptionKeyName -EncryptionKeyVersion $encryptionKeyVersion `
175176
-EncryptionVaultUri $encryptionVaultUri
176177

‎src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightAutoscaleTests.ps1‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,12 @@ function Test-AutoscaleRelatedCommands{
2222
try
2323
{
2424
# prepare parameter for creating parameter
25-
$params= Prepare-ClusterCreateParameterForWASB
25+
$params= Prepare-ClusterCreateParameter
2626

2727
# test create cluster
2828
$cluster = New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
2929
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
30-
-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
30+
-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
3131
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
3232
-MinSupportedTlsVersion $params.minSupportedTlsVersion
3333

‎src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightClusterTests.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,5 +70,12 @@ public void TestCreateClusterWithScheduleBasedAutoscale()
7070
{
7171
TestController.NewInstance.RunPowerShellTest(_logger, "Test-CreateClusterWithScheduleBasedAutoscale");
7272
}
73+
74+
[Fact]
75+
[Trait(Category.AcceptanceType, Category.CheckIn)]
76+
public void TestCreateClusterWithKafkaRestProxy()
77+
{
78+
TestController.NewInstance.RunPowerShellTest(_logger, "Test-CreateClusterWithKafkaRestProxy");
79+
}
7380
}
7481
}

‎src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightClusterTests.ps1‎

Lines changed: 47 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,12 @@ function Test-ClusterRelatedCommands{
2323
try
2424
{
2525
# prepare parameter for creating parameter
26-
$params= Prepare-ClusterCreateParameterForWASB
26+
$params= Prepare-ClusterCreateParameter
2727

2828
# test create cluster
2929
$cluster = New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
3030
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
31-
-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
31+
-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
3232
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
3333
-MinSupportedTlsVersion $params.minSupportedTlsVersion
3434

@@ -104,13 +104,13 @@ function Test-CreateClusterWithEncryptionInTransit{
104104
try
105105
{
106106
# prepare parameter for creating parameter
107-
$params= Prepare-ClusterCreateParameterForWASB -Location "South Central US"
107+
$params= Prepare-ClusterCreateParameter -Location "South Central US"
108108
$encryptionInTransit=$true
109109

110110
# create cluster
111111
$cluster=New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
112112
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
113-
-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
113+
-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
114114
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
115115
-MinSupportedTlsVersion $params.minSupportedTlsVersion -EncryptionInTransit $encryptionInTransit
116116

@@ -136,7 +136,7 @@ function Test-CreateClusterWithEncryptionAtHost{
136136
try
137137
{
138138
# prepare parameter for creating parameter
139-
$params= Prepare-ClusterCreateParameterForWASB -location "South Central US"
139+
$params= Prepare-ClusterCreateParameter -location "South Central US"
140140
$encryptionAtHost=$true
141141
$workerNodeSize="Standard_DS14_v2"
142142
$headNodeSize="Standard_DS14_v2"
@@ -146,7 +146,7 @@ function Test-CreateClusterWithEncryptionAtHost{
146146
$cluster=New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
147147
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
148148
-WorkerNodeSize $workerNodeSize -HeadNodeSize $headNodeSize -ZookeeperNodeSize $zookeeperNodeSize `
149-
-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
149+
-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
150150
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
151151
-MinSupportedTlsVersion $params.minSupportedTlsVersion -EncryptionAtHost $encryptionAtHost
152152

@@ -172,15 +172,15 @@ function Test-CreateClusterWithLoadBasedAutoscale{
172172
try
173173
{
174174
# prepare parameter for creating parameter
175-
$params= Prepare-ClusterCreateParameterForWASB -location "East US"
175+
$params= Prepare-ClusterCreateParameter -location "East US"
176176

177177
# create autoscale cofiguration
178178
$autoscaleConfiguration=New-AzHDInsightClusterAutoscaleConfiguration -MinWorkerNodeCount 4 -MaxWorkerNodeCount 5
179179

180180
# create cluster with load-based autoscale
181181
$cluster=New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
182182
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
183-
-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
183+
-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
184184
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
185185
-MinSupportedTlsVersion $params.minSupportedTlsVersion -Version 4.0 `
186186
-AutoscaleConfiguration $autoscaleConfiguration
@@ -208,7 +208,7 @@ function Test-CreateClusterWithScheduleBasedAutoscale{
208208
try
209209
{
210210
# prepare parameter for creating parameter
211-
$params= Prepare-ClusterCreateParameterForWASB -location "East US"
211+
$params= Prepare-ClusterCreateParameter -location "East US"
212212

213213
# create autoscale schedule condition
214214
$condition1=New-AzHDInsightClusterAutoscaleScheduleCondition -Time "09:00" -WorkerNodeCount 4 -Day Monday,Tuesday
@@ -221,7 +221,7 @@ function Test-CreateClusterWithScheduleBasedAutoscale{
221221
# create cluster with schedule-based autoscale
222222
$cluster=New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
223223
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
224-
-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
224+
-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
225225
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
226226
-MinSupportedTlsVersion $params.minSupportedTlsVersion -Version 4.0 `
227227
-AutoscaleConfiguration $autoscaleConfiguration
@@ -238,3 +238,40 @@ function Test-CreateClusterWithScheduleBasedAutoscale{
238238
Remove-AzResourceGroup -ResourceGroupName $cluster.ResourceGroup
239239
}
240240
}
241+
242+
<#
243+
.SYNOPSIS
244+
Test Create Azure HDInsight Cluster with kafka rest proxy.
245+
#>
246+
247+
function Test-CreateClusterWithKafkaRestProxy{
248+
# Create some resources that will be used throughout test
249+
try
250+
{
251+
# prepare parameter for creating parameter
252+
$params= Prepare-ClusterCreateParameter -location "South Central US" -clusterType Kafka
253+
$kafkaClientGroupName="FakeClientGroup"
254+
$kafkaClientGroupId="00000000-0000-0000-0000-000000000000"
255+
$disksPerWorkerNode=2
256+
257+
# test create cluster
258+
$cluster = New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
259+
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
260+
-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
261+
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
262+
-MinSupportedTlsVersion $params.minSupportedTlsVersion -KafkaClientGroupId $kafkaClientGroupId `
263+
-KafkaClientGroupName $kafkaClientGroupName -DisksPerWorkerNode $disksPerWorkerNode `
264+
-KafkaManagementNodeSize Standard_D4_v2
265+
266+
Assert-NotNull $cluster
267+
#test Get-AzHDInsightCluster
268+
$resultCluster = Get-AzHDInsightCluster -ClusterName $cluster.Name
269+
Assert-AreEqual $resultCluster.Name $cluster.Name
270+
}
271+
finally
272+
{
273+
# Delete cluster and resource group
274+
Remove-AzHDInsightCluster -ClusterName $cluster.Name
275+
Remove-AzResourceGroup -ResourceGroupName $cluster.ResourceGroup
276+
}
277+
}

‎src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightHostTests.ps1‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,12 @@ function Test-HostRelatedCommands{
2323
try
2424
{
2525
# prepare parameter for creating parameter
26-
$params= Prepare-ClusterCreateParameterForWASB
26+
$params= Prepare-ClusterCreateParameter
2727

2828
# create cluster that will be used throughout test
2929
$cluster = New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
3030
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
31-
-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
31+
-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
3232
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
3333
-MinSupportedTlsVersion $params.minSupportedTlsVersion
3434
Assert-NotNull $cluster

‎src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightSetGatewayCredentialTest.ps1‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,12 @@ function Test-SetGatewayCredential{
2222
try
2323
{
2424
# prepare parameter for creating parameter
25-
$params= Prepare-ClusterCreateParameterForWASB
25+
$params= Prepare-ClusterCreateParameter
2626

2727
# create cluster that will be used throughout test
2828
$cluster = New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
2929
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
30-
-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
30+
-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
3131
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
3232
-MinSupportedTlsVersion $params.minSupportedTlsVersion
3333

0 commit comments

Comments
 (0)