Skip to content

Commit 4fd46ff

Browse files
aim-for-betterZhenyu Zhou
aim-for-better
and
Zhenyu Zhou
authored
[HDInsight] breaking change: Support ADLSGen2 MSI, Kafka Rest Proxy, ESP and HIB feature (#13230)
* Deprecate Parameters * 1. Support ADLSGen2 MSI: Remove the Default Prefix of Storage parameter, add StorageFileSystem, StorageAccountManagedIdentity parameters 2. Support KafkaRestProxy: add new parameters: KafkaClientGroupName, KafkaClientGroupId, KafkaManagementNodeSize 3. Support ESP and HIB: Change Domin to DominResourceId and -EnableIDBroker switch parameter 4. Require Storage Resource Id instead name * Add CredScanSuppression because the resource has been deleted * Fix Static analysis errors Co-authored-by: Zhenyu Zhou <[email protected]>
1 parent 2f832e2 commit 4fd46ff

File tree

47 files changed

+5059
-838
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+5059
-838
lines changed

‎src/HDInsight/HDInsight.Test/DataLakeStoreDefaultFilesystemTests.cs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ namespace Commands.HDInsight.Test.UnitTests
2929
public class DataLakeStoreDefaultFilesystemTests : HDInsightTestBase
3030
{
3131
private NewAzureHDInsightClusterCommand cmdlet;
32-
private const string StorageName = "dummystorage.azuredatalakestore.net";
32+
private const string StorageAccountResourceId = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/fakerg/providers/Microsoft.Storage/storageAccounts/dummystorage";
3333
private const int ClusterSize = 4;
3434
private Guid ObjectId = new Guid("11111111-1111-1111-1111-111111111111");
3535
private Guid ApplicationId = new Guid("11111111-1111-1111-1111-111111111111");
@@ -66,7 +66,7 @@ public void CanCreateClusterConfigWithDataLakeStoreParameters()
6666
CertificateFilePath = Certificate,
6767
AadTenantId = AadTenantId,
6868
CertificatePassword = CertificatePassword,
69-
DefaultStorageAccountName = StorageName
69+
StorageAccountResourceId = StorageAccountResourceId
7070
};
7171

7272
newclusteridentitycmdlet.ExecuteCmdlet();
@@ -80,7 +80,7 @@ public void CanCreateClusterConfigWithDataLakeStoreParameters()
8080
c.ObjectId == ObjectId &&
8181
c.ApplicationId == ApplicationId &&
8282
c.CertificateFilePath == Certificate &&
83-
c.DefaultStorageAccountName == StorageName
83+
c.StorageAccountResourceId == StorageAccountResourceId
8484
)),
8585
Times.Once);
8686
}
@@ -98,7 +98,7 @@ public void CanCreateDataLakeClusterWithCertificateFileContents()
9898
CertificateFileContents = CertificateFileContents,
9999
AadTenantId = AadTenantId,
100100
CertificatePassword = CertificatePassword,
101-
DefaultStorageAccountName = StorageName
101+
StorageAccountResourceId = StorageAccountResourceId
102102
};
103103

104104
clusterIdentityCmdlet.ExecuteCmdlet();
@@ -112,7 +112,7 @@ public void CanCreateDataLakeClusterWithCertificateFileContents()
112112
c.ObjectId == ObjectId &&
113113
c.ApplicationId == ApplicationId &&
114114
c.CertificateFileContents == CertificateFileContents &&
115-
c.DefaultStorageAccountName == StorageName
115+
c.StorageAccountResourceId == StorageAccountResourceId
116116
)),
117117
Times.Once);
118118
}

‎src/HDInsight/HDInsight.Test/ScenarioTests/Common.ps1‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ class ClusterCommonCreateParameter{
7373
[string] $clusterName
7474
[string] $location
7575
[string] $resourceGroupName
76-
[string] $storageAccountName
76+
[string] $storageAccountResourceId
7777
[string] $clusterType
7878
[int] $clusterSizeInNodes
7979
[string] $storageAccountKey
@@ -82,13 +82,13 @@ class ClusterCommonCreateParameter{
8282
[string] $minSupportedTlsVersion
8383

8484
ClusterCommonCreateParameter([string] $clusterName, [string] $location, [string] $resourceGroupName,
85-
[string] $storageAccountName, [string] $clusterType, [int] $clusterSizeInNodes,
85+
[string] $storageAccountResourceId, [string] $clusterType, [int] $clusterSizeInNodes,
8686
[string] $storageAccountKey, [System.Management.Automation.PSCredential] $httpCredential,
8787
[System.Management.Automation.PSCredential] $sshCredential, [string] $minSupportedTlsVersion){
8888
$this.clusterName=$clusterName
8989
$this.location=$location
9090
$this.resourceGroupName=$resourceGroupName
91-
$this.storageAccountName=$storageAccountName
91+
$this.storageAccountResourceId=$storageAccountResourceId
9292
$this.clusterType=$clusterType
9393
$this.clusterSizeInNodes=$clusterSizeInNodes
9494
$this.storageAccountKey=$storageAccountKey
@@ -100,9 +100,9 @@ class ClusterCommonCreateParameter{
100100

101101
<#
102102
.SYNOPSIS
103-
Create Common Parameter for creating cluster.
103+
Create Common Parameter with WASB for creating cluster.
104104
#>
105-
function Prepare-ClusterCreateParameterForWASB{
105+
function Prepare-ClusterCreateParameter{
106106
param(
107107
[string] $clusterName="hdi-ps-test",
108108
[string] $location="East US",
@@ -118,8 +118,9 @@ function Prepare-ClusterCreateParameterForWASB{
118118

119119
$storageAccountName=Generate-StorageAccountName($storageAccountName)
120120

121-
$storageAccount= New-AzStorageAccount -ResourceGroupName $resourceGroupName -Location $location -Name $storageAccountName -TypeString "Standard_RAGRS"
121+
$storageAccount= New-AzStorageAccount -ResourceGroupName $resourceGroupName -Location $location -Name $storageAccountName -TypeString Standard_RAGRS
122122

123+
$storageAccountResourceId=$storageAccount.Id
123124
$storageAccountKey=Get-AzStorageAccountKey -ResourceGroupName $resourceGroupName -Name $storageAccountName
124125
$storageAccountKey=$storageAccountKey[0].Value
125126

@@ -131,9 +132,9 @@ function Prepare-ClusterCreateParameterForWASB{
131132
$httpCredential=New-Object System.Management.Automation.PSCredential($httpUser, $httpPassword)
132133
$sshCredential=New-Object System.Management.Automation.PSCredential($sshUser, $sshPassword)
133134

134-
$clusterSizeInNodes=2
135+
$clusterSizeInNodes=3
135136
$minSupportedTlsVersion="1.2"
136-
return [ClusterCommonCreateParameter]::new($clusterName, $location, $resourceGroupName,$storageAccountName,
137+
return [ClusterCommonCreateParameter]::new($clusterName, $location, $resourceGroupName,$storageAccountResourceId,
137138
$clusterType, $clusterSizeInNodes,$storageAccountKey, $httpCredential,
138139
$sshCredential, $minSupportedTlsVersion)
139140
}
@@ -151,7 +152,7 @@ function Create-CMKCluster{
151152
[string] $keyName="key-ps-cmktest"
152153
)
153154

154-
$params=Prepare-ClusterCreateParameterForWASB -clusterName $clusterName -location $location
155+
$params=Prepare-ClusterCreateParameter -clusterName $clusterName -location $location
155156

156157
# new user-assigned identity
157158
$assignedIdentity= New-AzUserAssignedIdentity -ResourceGroupName $params.resourceGroupName -Name $assignedIdentityName
@@ -169,8 +170,8 @@ function Create-CMKCluster{
169170
$encryptionKeyName=$encryptionKey.Name
170171
# new hdi cluster with cmk
171172
$cluster=New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName -ClusterName $params.clusterName `
172-
-ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType -DefaultStorageAccountName $params.storageAccountName `
173-
-DefaultStorageAccountKey $params.storageAccountKey -HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
173+
-ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType -StorageAccountResourceId $params.storageAccountResourceId `
174+
-StorageAccountKey $params.storageAccountKey -HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
174175
-AssignedIdentity $assignedIdentityId -EncryptionKeyName $encryptionKeyName -EncryptionKeyVersion $encryptionKeyVersion `
175176
-EncryptionVaultUri $encryptionVaultUri
176177

‎src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightAutoscaleTests.ps1‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,12 @@ function Test-AutoscaleRelatedCommands{
2222
try
2323
{
2424
# prepare parameter for creating parameter
25-
$params= Prepare-ClusterCreateParameterForWASB
25+
$params= Prepare-ClusterCreateParameter
2626

2727
# test create cluster
2828
$cluster = New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
2929
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
30-
-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
30+
-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
3131
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
3232
-MinSupportedTlsVersion $params.minSupportedTlsVersion
3333

‎src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightClusterTests.cs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -50,13 +50,6 @@ public void TestCreateClusterWithEncryptionInTransit()
5050
TestController.NewInstance.RunPowerShellTest(_logger, "Test-CreateClusterWithEncryptionInTransit");
5151
}
5252

53-
[Fact]
54-
[Trait(Category.AcceptanceType, Category.CheckIn)]
55-
public void TestCreateClusterWithPrivateLink()
56-
{
57-
TestController.NewInstance.RunPowerShellTest(_logger, "Test-CreateClusterWithPrivateLink");
58-
}
59-
6053
[Fact]
6154
[Trait(Category.AcceptanceType, Category.CheckIn)]
6255
public void TestCreateClusterWithEncryptionAtHost()
@@ -77,5 +70,12 @@ public void TestCreateClusterWithScheduleBasedAutoscale()
7770
{
7871
TestController.NewInstance.RunPowerShellTest(_logger, "Test-CreateClusterWithScheduleBasedAutoscale");
7972
}
73+
74+
[Fact]
75+
[Trait(Category.AcceptanceType, Category.CheckIn)]
76+
public void TestCreateClusterWithKafkaRestProxy()
77+
{
78+
TestController.NewInstance.RunPowerShellTest(_logger, "Test-CreateClusterWithKafkaRestProxy");
79+
}
8080
}
8181
}

‎src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightClusterTests.ps1‎

Lines changed: 47 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,12 @@ function Test-ClusterRelatedCommands{
2323
try
2424
{
2525
# prepare parameter for creating parameter
26-
$params= Prepare-ClusterCreateParameterForWASB
26+
$params= Prepare-ClusterCreateParameter
2727

2828
# test create cluster
2929
$cluster = New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
3030
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
31-
-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
31+
-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
3232
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
3333
-MinSupportedTlsVersion $params.minSupportedTlsVersion
3434

@@ -104,13 +104,13 @@ function Test-CreateClusterWithEncryptionInTransit{
104104
try
105105
{
106106
# prepare parameter for creating parameter
107-
$params= Prepare-ClusterCreateParameterForWASB -Location "South Central US"
107+
$params= Prepare-ClusterCreateParameter -Location "South Central US"
108108
$encryptionInTransit=$true
109109

110110
# create cluster
111111
$cluster=New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
112112
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
113-
-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
113+
-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
114114
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
115115
-MinSupportedTlsVersion $params.minSupportedTlsVersion -EncryptionInTransit $encryptionInTransit
116116

@@ -125,45 +125,6 @@ function Test-CreateClusterWithEncryptionInTransit{
125125
}
126126
}
127127

128-
<#
129-
.SYNOPSIS
130-
Test Create Azure HDInsight Cluster which Private Link
131-
#>
132-
133-
function Test-CreateClusterWithPrivateLink{
134-
135-
# Create some resources that will be used throughout test
136-
try
137-
{
138-
# prepare parameter for creating parameter
139-
$params= Prepare-ClusterCreateParameterForWASB -location "South Central US"
140-
141-
# Prepare virtual network
142-
$vnetName=Generate-Name("hdi-ps-vnet")
143-
$vnet=Create-VnetkWithSubnet -location $params.location -resourceGroupName $params.resourceGroupName `
144-
-vnetName $vnetName -subnetPrivateLinkServiceNetworkPoliciesFlag $false
145-
146-
# create cluster
147-
$cluster=New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
148-
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
149-
-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
150-
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
151-
-MinSupportedTlsVersion $params.minSupportedTlsVersion `
152-
-VirtualNetworkId $vnet.Id -SubnetName $vnet.Subnets[0].Id `
153-
-PublicNetworkAccessType OutboundOnly -OutboundPublicNetworkAccessType PublicLoadBalancer
154-
155-
Assert-AreEqual $cluster.PublicNetworkAccessType OutboundOnly
156-
Assert-AreEqual $cluster.OutboundPublicNetworkAccessType PublicLoadBalancer
157-
158-
}
159-
finally
160-
{
161-
# Delete cluster and resource group
162-
Remove-AzHDInsightCluster -ClusterName $cluster.Name
163-
Remove-AzResourceGroup -ResourceGroupName $cluster.ResourceGroup
164-
}
165-
}
166-
167128
<#
168129
.SYNOPSIS
169130
Test Create Azure HDInsight Cluster which enalbes Encryption At Host
@@ -175,7 +136,7 @@ function Test-CreateClusterWithEncryptionAtHost{
175136
try
176137
{
177138
# prepare parameter for creating parameter
178-
$params= Prepare-ClusterCreateParameterForWASB -location "South Central US"
139+
$params= Prepare-ClusterCreateParameter -location "South Central US"
179140
$encryptionAtHost=$true
180141
$workerNodeSize="Standard_DS14_v2"
181142
$headNodeSize="Standard_DS14_v2"
@@ -185,7 +146,7 @@ function Test-CreateClusterWithEncryptionAtHost{
185146
$cluster=New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
186147
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
187148
-WorkerNodeSize $workerNodeSize -HeadNodeSize $headNodeSize -ZookeeperNodeSize $zookeeperNodeSize `
188-
-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
149+
-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
189150
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
190151
-MinSupportedTlsVersion $params.minSupportedTlsVersion -EncryptionAtHost $encryptionAtHost
191152

@@ -211,15 +172,15 @@ function Test-CreateClusterWithLoadBasedAutoscale{
211172
try
212173
{
213174
# prepare parameter for creating parameter
214-
$params= Prepare-ClusterCreateParameterForWASB -location "East US"
175+
$params= Prepare-ClusterCreateParameter -location "East US"
215176

216177
# create autoscale cofiguration
217178
$autoscaleConfiguration=New-AzHDInsightClusterAutoscaleConfiguration -MinWorkerNodeCount 4 -MaxWorkerNodeCount 5
218179

219180
# create cluster with load-based autoscale
220181
$cluster=New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
221182
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
222-
-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
183+
-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
223184
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
224185
-MinSupportedTlsVersion $params.minSupportedTlsVersion -Version 4.0 `
225186
-AutoscaleConfiguration $autoscaleConfiguration
@@ -247,7 +208,7 @@ function Test-CreateClusterWithScheduleBasedAutoscale{
247208
try
248209
{
249210
# prepare parameter for creating parameter
250-
$params= Prepare-ClusterCreateParameterForWASB -location "East US"
211+
$params= Prepare-ClusterCreateParameter -location "East US"
251212

252213
# create autoscale schedule condition
253214
$condition1=New-AzHDInsightClusterAutoscaleScheduleCondition -Time "09:00" -WorkerNodeCount 4 -Day Monday,Tuesday
@@ -260,7 +221,7 @@ function Test-CreateClusterWithScheduleBasedAutoscale{
260221
# create cluster with schedule-based autoscale
261222
$cluster=New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
262223
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
263-
-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
224+
-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
264225
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
265226
-MinSupportedTlsVersion $params.minSupportedTlsVersion -Version 4.0 `
266227
-AutoscaleConfiguration $autoscaleConfiguration
@@ -277,3 +238,40 @@ function Test-CreateClusterWithScheduleBasedAutoscale{
277238
Remove-AzResourceGroup -ResourceGroupName $cluster.ResourceGroup
278239
}
279240
}
241+
242+
<#
243+
.SYNOPSIS
244+
Test Create Azure HDInsight Cluster with kafka rest proxy.
245+
#>
246+
247+
function Test-CreateClusterWithKafkaRestProxy{
248+
# Create some resources that will be used throughout test
249+
try
250+
{
251+
# prepare parameter for creating parameter
252+
$params= Prepare-ClusterCreateParameter -location "South Central US" -clusterType Kafka
253+
$kafkaClientGroupName="FakeClientGroup"
254+
$kafkaClientGroupId="00000000-0000-0000-0000-000000000000"
255+
$disksPerWorkerNode=2
256+
257+
# test create cluster
258+
$cluster = New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
259+
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
260+
-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
261+
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
262+
-MinSupportedTlsVersion $params.minSupportedTlsVersion -KafkaClientGroupId $kafkaClientGroupId `
263+
-KafkaClientGroupName $kafkaClientGroupName -DisksPerWorkerNode $disksPerWorkerNode `
264+
-KafkaManagementNodeSize Standard_D4_v2
265+
266+
Assert-NotNull $cluster
267+
#test Get-AzHDInsightCluster
268+
$resultCluster = Get-AzHDInsightCluster -ClusterName $cluster.Name
269+
Assert-AreEqual $resultCluster.Name $cluster.Name
270+
}
271+
finally
272+
{
273+
# Delete cluster and resource group
274+
Remove-AzHDInsightCluster -ClusterName $cluster.Name
275+
Remove-AzResourceGroup -ResourceGroupName $cluster.ResourceGroup
276+
}
277+
}

‎src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightHostTests.ps1‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,12 @@ function Test-HostRelatedCommands{
2323
try
2424
{
2525
# prepare parameter for creating parameter
26-
$params= Prepare-ClusterCreateParameterForWASB
26+
$params= Prepare-ClusterCreateParameter
2727

2828
# create cluster that will be used throughout test
2929
$cluster = New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
3030
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
31-
-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
31+
-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
3232
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
3333
-MinSupportedTlsVersion $params.minSupportedTlsVersion
3434
Assert-NotNull $cluster

‎src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightSetGatewayCredentialTest.ps1‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,12 @@ function Test-SetGatewayCredential{
2222
try
2323
{
2424
# prepare parameter for creating parameter
25-
$params= Prepare-ClusterCreateParameterForWASB
25+
$params= Prepare-ClusterCreateParameter
2626

2727
# create cluster that will be used throughout test
2828
$cluster = New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
2929
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
30-
-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
30+
-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
3131
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
3232
-MinSupportedTlsVersion $params.minSupportedTlsVersion
3333

0 commit comments

Comments
 (0)